2023-07-22 12:33:29,276 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e 2023-07-22 12:33:29,291 main DEBUG Took 0.011940 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-07-22 12:33:29,291 main DEBUG PluginManager 'Core' found 129 plugins 2023-07-22 12:33:29,292 main DEBUG PluginManager 'Level' found 0 plugins 2023-07-22 12:33:29,293 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-07-22 12:33:29,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,305 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-07-22 12:33:29,320 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,322 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,323 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,325 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,326 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,327 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,327 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,328 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,329 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,329 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,329 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,330 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,331 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:33:29,332 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,332 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-07-22 12:33:29,334 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:33:29,337 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-07-22 12:33:29,339 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-07-22 12:33:29,340 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-07-22 12:33:29,341 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-07-22 12:33:29,342 main DEBUG PluginManager 'Converter' found 47 plugins 2023-07-22 12:33:29,358 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-07-22 12:33:29,362 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-07-22 12:33:29,366 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-07-22 12:33:29,366 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-07-22 12:33:29,366 main DEBUG createAppenders(={Console}) 2023-07-22 12:33:29,367 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e initialized 2023-07-22 12:33:29,367 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e 2023-07-22 12:33:29,368 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e OK. 2023-07-22 12:33:29,368 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-07-22 12:33:29,368 main DEBUG OutputStream closed 2023-07-22 12:33:29,369 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-07-22 12:33:29,369 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-07-22 12:33:29,369 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@72a7c7e0 OK 2023-07-22 12:33:29,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-07-22 12:33:29,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-07-22 12:33:29,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-07-22 12:33:29,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-07-22 12:33:29,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-07-22 12:33:29,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-07-22 12:33:29,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-07-22 12:33:29,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-07-22 12:33:29,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-07-22 12:33:29,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-07-22 12:33:29,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-07-22 12:33:29,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-07-22 12:33:29,426 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-07-22 12:33:29,426 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-07-22 12:33:29,427 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-07-22 12:33:29,427 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-07-22 12:33:29,427 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-07-22 12:33:29,428 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-07-22 12:33:29,432 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22 12:33:29,432 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@e7edb54) with optional ClassLoader: null 2023-07-22 12:33:29,432 main DEBUG Shutdown hook enabled. Registering a new one. 2023-07-22 12:33:29,433 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@e7edb54] started OK. 2023-07-22T12:33:29,466 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures timeout: 13 mins 2023-07-22 12:33:29,469 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-07-22 12:33:29,470 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22T12:33:29,786 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c 2023-07-22T12:33:29,790 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:33:29,792 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:29,794 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:33:29,797 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed, deleteOnExit=true 2023-07-22T12:33:29,797 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-07-22T12:33:29,797 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/test.cache.data in system properties and HBase conf 2023-07-22T12:33:29,798 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:33:29,798 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:33:29,799 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:33:29,799 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:33:29,799 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:33:29,893 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-07-22T12:33:30,306 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:33:30,310 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:33:30,311 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:33:30,311 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:33:30,311 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:33:30,311 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:33:30,312 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:33:30,312 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:33:30,312 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:33:30,313 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:33:30,313 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:33:30,313 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:33:30,313 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:33:30,314 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:33:30,314 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:33:30,823 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:33:30,828 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:33:31,104 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-07-22T12:33:31,271 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@d31268e via org.mortbay.log.Slf4jLog 2023-07-22T12:33:31,286 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:33:31,324 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:33:31,360 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir/Jetty_localhost_46169_hdfs____n8nwkc/webapp 2023-07-22T12:33:31,490 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46169 2023-07-22T12:33:31,511 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:33:31,511 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:33:32,015 WARN [Listener at localhost/33927 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:33:32,079 WARN [Listener at localhost/33927 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:33:32,100 WARN [Listener at localhost/33927 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:33:32,106 INFO [Listener at localhost/33927 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:33:32,112 INFO [Listener at localhost/33927 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir/Jetty_localhost_36089_datanode____.secze3/webapp 2023-07-22T12:33:32,216 INFO [Listener at localhost/33927 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36089 2023-07-22T12:33:32,484 WARN [Listener at localhost/43917 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:33:32,782 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:33:32,952 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x253f97124ae9877e: Processing first storage report for DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b from datanode 0a0352df-90fc-48e7-b46e-4ee4b5b1fc11 2023-07-22T12:33:32,953 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x253f97124ae9877e: from storage DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b node DatanodeRegistration(127.0.0.1:34885, datanodeUuid=0a0352df-90fc-48e7-b46e-4ee4b5b1fc11, infoPort=45283, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=1120339800;c=1690029210898), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-07-22T12:33:32,953 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x253f97124ae9877e: Processing first storage report for DS-8e5f3236-afa0-45cb-a344-2ef8f4eb210c from datanode 0a0352df-90fc-48e7-b46e-4ee4b5b1fc11 2023-07-22T12:33:32,954 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x253f97124ae9877e: from storage DS-8e5f3236-afa0-45cb-a344-2ef8f4eb210c node DatanodeRegistration(127.0.0.1:34885, datanodeUuid=0a0352df-90fc-48e7-b46e-4ee4b5b1fc11, infoPort=45283, infoSecurePort=0, ipcPort=43917, storageInfo=lv=-57;cid=testClusterID;nsid=1120339800;c=1690029210898), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:33:33,031 DEBUG [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c 2023-07-22T12:33:33,116 INFO [Listener at localhost/43917 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/zookeeper_0, clientPort=56341, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-07-22T12:33:33,133 INFO [Listener at localhost/43917 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=56341 2023-07-22T12:33:33,140 INFO [Listener at localhost/43917 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:33,142 INFO [Listener at localhost/43917 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:33,781 INFO [Listener at localhost/43917 {}] util.FSUtils(461): Created version file at hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 with version=8 2023-07-22T12:33:33,781 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/hbase-staging 2023-07-22T12:33:33,798 DEBUG [Listener at localhost/43917 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:33:33,798 DEBUG [Listener at localhost/43917 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:33:33,798 DEBUG [Listener at localhost/43917 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:33:33,798 DEBUG [Listener at localhost/43917 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:33:34,012 DEBUG [Listener at localhost/43917 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-07-22T12:33:34,165 INFO [Listener at localhost/43917 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-07-22T12:33:34,800 INFO [Listener at localhost/43917 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:33:34,847 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:33:34,847 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:33:34,848 INFO [Listener at localhost/43917 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:33:34,848 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:33:34,848 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:33:35,008 INFO [Listener at localhost/43917 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:33:35,113 DEBUG [Listener at localhost/43917 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-07-22T12:33:35,119 INFO [Listener at localhost/43917 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:33:35,168 DEBUG [Listener at localhost/43917 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 15591 (auto-detected) 2023-07-22T12:33:35,170 DEBUG [Listener at localhost/43917 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-07-22T12:33:35,208 INFO [Listener at localhost/43917 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:44705 2023-07-22T12:33:35,222 INFO [Listener at localhost/43917 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:35,224 INFO [Listener at localhost/43917 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:35,242 INFO [Listener at localhost/43917 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:44705 connecting to ZooKeeper ensemble=127.0.0.1:56341 2023-07-22T12:33:35,289 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:447050x0, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:33:35,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:44705-0x1000060a6040000 connected 2023-07-22T12:33:35,319 DEBUG [Listener at localhost/43917 {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:33:35,322 DEBUG [Listener at localhost/43917 {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:33:35,326 DEBUG [Listener at localhost/43917 {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:33:35,336 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44705 2023-07-22T12:33:35,337 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44705 2023-07-22T12:33:35,337 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44705 2023-07-22T12:33:35,338 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44705 2023-07-22T12:33:35,338 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44705 2023-07-22T12:33:35,393 INFO [Listener at localhost/43917 {}] log.Log(170): Logging initialized @6776ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-07-22T12:33:35,516 INFO [Listener at localhost/43917 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:33:35,517 INFO [Listener at localhost/43917 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:33:35,517 INFO [Listener at localhost/43917 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:33:35,520 INFO [Listener at localhost/43917 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:33:35,520 INFO [Listener at localhost/43917 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:33:35,520 INFO [Listener at localhost/43917 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:33:35,523 INFO [Listener at localhost/43917 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:33:35,653 INFO [Listener at localhost/43917 {}] http.HttpServer(1219): Jetty bound to port 45139 2023-07-22T12:33:35,655 INFO [Listener at localhost/43917 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:33:35,687 INFO [Listener at localhost/43917 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:33:35,691 INFO [Listener at localhost/43917 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@20cd50cf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:33:35,692 INFO [Listener at localhost/43917 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:33:35,693 INFO [Listener at localhost/43917 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@55284cf3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:33:35,757 INFO [Listener at localhost/43917 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:33:35,771 INFO [Listener at localhost/43917 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:33:35,771 INFO [Listener at localhost/43917 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:33:35,773 INFO [Listener at localhost/43917 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:33:35,780 INFO [Listener at localhost/43917 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:33:36,921 INFO [Listener at localhost/43917 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3ce1cd2d{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:33:36,932 INFO [Listener at localhost/43917 {}] server.AbstractConnector(333): Started ServerConnector@2d49367d{HTTP/1.1, (http/1.1)}{0.0.0.0:45139} 2023-07-22T12:33:36,932 INFO [Listener at localhost/43917 {}] server.Server(415): Started @8316ms 2023-07-22T12:33:36,936 INFO [Listener at localhost/43917 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203, hbase.cluster.distributed=false 2023-07-22T12:33:36,994 INFO [Listener at localhost/43917 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:33:36,994 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:33:36,995 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:33:36,995 INFO [Listener at localhost/43917 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:33:36,995 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:33:36,995 INFO [Listener at localhost/43917 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:33:37,003 INFO [Listener at localhost/43917 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:33:37,004 INFO [Listener at localhost/43917 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:33:37,006 INFO [Listener at localhost/43917 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:42129 2023-07-22T12:33:37,008 INFO [Listener at localhost/43917 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:33:37,015 DEBUG [Listener at localhost/43917 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:33:37,016 INFO [Listener at localhost/43917 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:37,019 INFO [Listener at localhost/43917 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:37,021 INFO [Listener at localhost/43917 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:42129 connecting to ZooKeeper ensemble=127.0.0.1:56341 2023-07-22T12:33:37,024 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:421290x0, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:33:37,025 DEBUG [Listener at localhost/43917 {}] zookeeper.ZKUtil(113): regionserver:421290x0, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:33:37,025 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:42129-0x1000060a6040001 connected 2023-07-22T12:33:37,026 DEBUG [Listener at localhost/43917 {}] zookeeper.ZKUtil(113): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:33:37,027 DEBUG [Listener at localhost/43917 {}] zookeeper.ZKUtil(113): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:33:37,027 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42129 2023-07-22T12:33:37,028 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42129 2023-07-22T12:33:37,028 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42129 2023-07-22T12:33:37,028 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42129 2023-07-22T12:33:37,029 DEBUG [Listener at localhost/43917 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42129 2023-07-22T12:33:37,033 INFO [Listener at localhost/43917 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:33:37,033 INFO [Listener at localhost/43917 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:33:37,033 INFO [Listener at localhost/43917 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:33:37,034 INFO [Listener at localhost/43917 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:33:37,034 INFO [Listener at localhost/43917 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:33:37,034 INFO [Listener at localhost/43917 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:33:37,035 INFO [Listener at localhost/43917 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:33:37,037 INFO [Listener at localhost/43917 {}] http.HttpServer(1219): Jetty bound to port 42195 2023-07-22T12:33:37,037 INFO [Listener at localhost/43917 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:33:37,039 INFO [Listener at localhost/43917 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:33:37,039 INFO [Listener at localhost/43917 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@15f06e61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:33:37,039 INFO [Listener at localhost/43917 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:33:37,040 INFO [Listener at localhost/43917 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@124c4546{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:33:37,050 INFO [Listener at localhost/43917 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:33:37,051 INFO [Listener at localhost/43917 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:33:37,051 INFO [Listener at localhost/43917 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:33:37,052 INFO [Listener at localhost/43917 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:33:37,053 INFO [Listener at localhost/43917 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:33:37,057 INFO [Listener at localhost/43917 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5ead26ef{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:33:37,058 INFO [Listener at localhost/43917 {}] server.AbstractConnector(333): Started ServerConnector@7ec090de{HTTP/1.1, (http/1.1)}{0.0.0.0:42195} 2023-07-22T12:33:37,058 INFO [Listener at localhost/43917 {}] server.Server(415): Started @8441ms 2023-07-22T12:33:37,063 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:33:37,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@1a8be828{HTTP/1.1, (http/1.1)}{0.0.0.0:38407} 2023-07-22T12:33:37,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8449ms 2023-07-22T12:33:37,066 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:37,077 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:33:37,077 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:33:37,079 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:37,085 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:44705 2023-07-22T12:33:37,097 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:33:37,097 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:33:37,097 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:37,097 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:37,098 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:33:37,100 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,44705,1690029213996 from backup master directory 2023-07-22T12:33:37,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:33:37,103 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:37,103 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:33:37,104 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:33:37,105 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:33:37,105 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:37,109 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-07-22T12:33:37,111 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-07-22T12:33:37,193 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/hbase.id with ID: 85fb1299-d4ef-431b-9463-23566c4816a2 2023-07-22T12:33:37,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:33:37,250 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:37,250 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:37,293 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:37,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:33:37,306 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-07-22T12:33:37,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-07-22T12:33:37,321 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-07-22T12:33:37,321 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-07-22T12:33:37,322 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:33:37,333 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:33:37,406 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store 2023-07-22T12:33:37,830 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-07-22T12:33:37,831 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:37,831 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:33:37,831 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:33:37,831 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:33:37,831 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:33:37,831 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:33:37,831 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:33:37,832 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:33:37,833 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/.initializing 2023-07-22T12:33:37,834 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/WALs/jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:37,836 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:33:37,844 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,44705,1690029213996.regiongroup-0 2023-07-22T12:33:37,858 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44705%2C1690029213996.jenkins-hbase3.apache.org%2C44705%2C1690029213996.regiongroup-0, suffix=, logDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/WALs/jenkins-hbase3.apache.org,44705,1690029213996, archiveDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/oldWALs, maxLogs=10 2023-07-22T12:33:37,886 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/WALs/jenkins-hbase3.apache.org,44705,1690029213996/jenkins-hbase3.apache.org%2C44705%2C1690029213996.jenkins-hbase3.apache.org%2C44705%2C1690029213996.regiongroup-0.1690029217865, exclude list is [], retry=0 2023-07-22T12:33:37,917 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:33:37,925 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:33:38,003 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/WALs/jenkins-hbase3.apache.org,44705,1690029213996/jenkins-hbase3.apache.org%2C44705%2C1690029213996.jenkins-hbase3.apache.org%2C44705%2C1690029213996.regiongroup-0.1690029217865 2023-07-22T12:33:38,004 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:33:38,005 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:38,005 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:38,009 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,010 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:33:38,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:38,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:38,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:33:38,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:38,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:38,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:33:38,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:38,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:38,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:33:38,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:38,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:38,141 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,142 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,159 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:33:38,162 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:33:38,166 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:38,167 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9491023840, jitterRate=-0.11607952415943146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:33:38,173 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:33:38,174 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:33:38,206 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc41467, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:38,260 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:33:38,281 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:33:38,281 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:33:38,284 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:33:38,286 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-07-22T12:33:38,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-07-22T12:33:38,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:33:38,327 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:33:38,343 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-07-22T12:33:38,345 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-07-22T12:33:38,348 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:33:38,350 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:33:38,351 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-07-22T12:33:38,354 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:33:38,360 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:33:38,368 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-07-22T12:33:38,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:33:38,370 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-07-22T12:33:38,385 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:33:38,386 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-07-22T12:33:38,391 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:33:38,391 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:33:38,392 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,392 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,392 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,44705,1690029213996, sessionid=0x1000060a6040000, setting cluster-up flag (Was=false) 2023-07-22T12:33:38,401 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:33:38,409 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,409 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,416 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-07-22T12:33:38,418 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:38,421 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,421 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,426 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-07-22T12:33:38,428 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:38,432 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,432 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:38,438 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-07-22T12:33:38,439 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:38,471 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:42129 2023-07-22T12:33:38,472 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(997): ClusterId : 85fb1299-d4ef-431b-9463-23566c4816a2 2023-07-22T12:33:38,474 INFO [RS:0;jenkins-hbase3:42129 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:33:38,477 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:33:38,482 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:33:38,482 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:33:38,486 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:33:38,486 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:33:38,488 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:33:38,489 DEBUG [RS:0;jenkins-hbase3:42129 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cb2620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:38,491 DEBUG [RS:0;jenkins-hbase3:42129 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a17497a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:33:38,496 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:33:38,496 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:33:38,496 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:33:38,499 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,44705,1690029213996 with isa=jenkins-hbase3.apache.org/172.31.12.81:42129, startcode=1690029216993 2023-07-22T12:33:38,518 DEBUG [RS:0;jenkins-hbase3:42129 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:33:38,582 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:33:38,591 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:33:38,596 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:33:38,598 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50563, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:33:38,606 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,44705,1690029213996 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:33:38,611 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44705 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:33:38,612 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:33:38,612 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:33:38,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:33:38,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:33:38,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:33:38,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,613 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:33:38,614 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,626 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690029248626 2023-07-22T12:33:38,629 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:33:38,635 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:33:38,636 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:33:38,637 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:33:38,642 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:38,642 INFO [PEWorker-2 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:33:38,649 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:33:38,649 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:33:38,650 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:33:38,650 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:33:38,651 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:33:38,653 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-07-22T12:33:38,653 WARN [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-07-22T12:33:38,654 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,658 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:33:38,666 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:33:38,667 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:33:38,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x63d2f513 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:33:38,683 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75068529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:38,683 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:33:38,686 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:33:38,686 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:33:38,689 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029218688,5,FailOnTimeoutGroup] 2023-07-22T12:33:38,689 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029218689,5,FailOnTimeoutGroup] 2023-07-22T12:33:38,689 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,689 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:33:38,691 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,691 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,754 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,44705,1690029213996 with isa=jenkins-hbase3.apache.org/172.31.12.81:42129, startcode=1690029216993 2023-07-22T12:33:38,759 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44705 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:38,772 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:38,772 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:33927 2023-07-22T12:33:38,772 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=45139 2023-07-22T12:33:38,778 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:33:38,778 DEBUG [RS:0;jenkins-hbase3:42129 {}] zookeeper.ZKUtil(111): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:38,778 WARN [RS:0;jenkins-hbase3:42129 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:33:38,778 INFO [RS:0;jenkins-hbase3:42129 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-07-22T12:33:38,779 INFO [RS:0;jenkins-hbase3:42129 {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-07-22T12:33:38,779 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:38,783 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,42129,1690029216993] 2023-07-22T12:33:38,798 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:33:38,807 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:33:38,824 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:33:38,829 INFO [RS:0;jenkins-hbase3:42129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:33:38,829 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,830 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:33:38,840 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,841 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,841 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,841 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,841 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,842 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,842 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:33:38,842 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,842 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,843 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,843 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,843 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:33:38,843 DEBUG [RS:0;jenkins-hbase3:42129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:33:38,844 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,845 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,845 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,845 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,845 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42129,1690029216993-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:33:38,862 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:33:38,865 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,42129,1690029216993-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:38,881 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,42129,1690029216993 started 2023-07-22T12:33:38,881 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,42129,1690029216993, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:42129, sessionid=0x1000060a6040001 2023-07-22T12:33:38,882 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:33:38,882 DEBUG [RS:0;jenkins-hbase3:42129 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:38,882 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,42129,1690029216993' 2023-07-22T12:33:38,882 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-07-22T12:33:38,883 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-07-22T12:33:38,883 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:33:38,883 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:33:38,884 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,42129,1690029216993' 2023-07-22T12:33:38,884 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:33:38,884 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:33:38,885 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:33:38,885 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:33:38,885 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:33:38,885 DEBUG [RS:0;jenkins-hbase3:42129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:38,885 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,42129,1690029216993' 2023-07-22T12:33:38,885 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-07-22T12:33:38,886 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-07-22T12:33:38,886 DEBUG [RS:0;jenkins-hbase3:42129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:33:38,886 INFO [RS:0;jenkins-hbase3:42129 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:33:38,887 INFO [RS:0;jenkins-hbase3:42129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:33:38,998 INFO [RS:0;jenkins-hbase3:42129 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:33:38,998 INFO [RS:0;jenkins-hbase3:42129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,42129,1690029216993.regiongroup-0 2023-07-22T12:33:39,001 INFO [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0, suffix=, logDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993, archiveDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs, maxLogs=32 2023-07-22T12:33:39,014 DEBUG [RS:0;jenkins-hbase3:42129 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003, exclude list is [], retry=0 2023-07-22T12:33:39,020 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:33:39,024 INFO [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 2023-07-22T12:33:39,025 DEBUG [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:33:39,065 INFO [PEWorker-2 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:33:39,065 INFO [PEWorker-2 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:39,478 DEBUG [PEWorker-2 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:39,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:33:39,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:33:39,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:39,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:39,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:33:39,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:33:39,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:39,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:39,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:33:39,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:33:39,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:39,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:39,493 DEBUG [PEWorker-2 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740 2023-07-22T12:33:39,493 DEBUG [PEWorker-2 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740 2023-07-22T12:33:39,496 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:33:39,497 DEBUG [PEWorker-2 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:33:39,501 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:39,502 INFO [PEWorker-2 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10881610240, jitterRate=0.013428926467895508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:33:39,504 DEBUG [PEWorker-2 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:33:39,504 DEBUG [PEWorker-2 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:33:39,504 INFO [PEWorker-2 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:33:39,504 DEBUG [PEWorker-2 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:33:39,504 DEBUG [PEWorker-2 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:33:39,504 DEBUG [PEWorker-2 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:33:39,506 INFO [PEWorker-2 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:33:39,506 DEBUG [PEWorker-2 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:33:39,509 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:33:39,509 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:33:39,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:33:39,531 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:33:39,535 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:33:39,687 DEBUG [jenkins-hbase3:44705 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:33:39,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:39,703 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,42129,1690029216993, state=OPENING 2023-07-22T12:33:39,713 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:33:39,716 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:39,716 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:39,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:33:39,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:33:39,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:39,919 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:39,922 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:33:39,928 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52318, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:33:39,943 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:33:39,944 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-07-22T12:33:39,944 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-07-22T12:33:39,944 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:33:39,945 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:33:39,948 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta, suffix=.meta, logDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993, archiveDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs, maxLogs=32 2023-07-22T12:33:39,987 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta, exclude list is [], retry=0 2023-07-22T12:33:39,992 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:33:39,995 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta 2023-07-22T12:33:39,995 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:33:39,996 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:40,001 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:40,002 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:33:40,018 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:33:40,018 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:33:40,023 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:33:40,023 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:40,024 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:33:40,024 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:33:40,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:33:40,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:33:40,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:40,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:40,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:33:40,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:33:40,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:40,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:40,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:33:40,032 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:33:40,033 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:40,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:33:40,035 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740 2023-07-22T12:33:40,038 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740 2023-07-22T12:33:40,040 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:33:40,042 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:33:40,044 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11310932480, jitterRate=0.053412675857543945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:33:40,045 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:33:40,057 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690029219910 2023-07-22T12:33:40,076 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:33:40,076 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:33:40,078 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:40,080 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,42129,1690029216993, state=OPEN 2023-07-22T12:33:40,082 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:33:40,082 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:33:40,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:33:40,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:33:40,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:33:40,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,42129,1690029216993 in 362 msec 2023-07-22T12:33:40,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:33:40,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 570 msec 2023-07-22T12:33:40,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7210 sec 2023-07-22T12:33:40,227 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690029220227, completionTime=-1 2023-07-22T12:33:40,227 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:33:40,228 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:33:40,293 DEBUG [hconnection-0x3235c85-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:40,296 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33294, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:40,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:33:40,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690029280312 2023-07-22T12:33:40,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690029340312 2023-07-22T12:33:40,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 84 msec 2023-07-22T12:33:40,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44705,1690029213996-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:40,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44705,1690029213996-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:40,337 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44705,1690029213996-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:40,339 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:44705, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:40,339 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:33:40,345 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:33:40,359 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:33:40,361 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:33:40,370 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:33:40,373 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:40,375 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:40,377 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:40,796 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 855ade4dc43fb86bf7ef486829b4203f, NAME => 'hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:41,208 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:41,208 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 855ade4dc43fb86bf7ef486829b4203f, disabling compactions & flushes 2023-07-22T12:33:41,208 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,208 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,208 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. after waiting 0 ms 2023-07-22T12:33:41,208 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,208 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,208 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 855ade4dc43fb86bf7ef486829b4203f: 2023-07-22T12:33:41,210 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:41,224 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690029221212"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029221212"}]},"ts":"1690029221212"} 2023-07-22T12:33:41,261 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:41,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:41,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029221263"}]},"ts":"1690029221263"} 2023-07-22T12:33:41,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:33:41,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=855ade4dc43fb86bf7ef486829b4203f, ASSIGN}] 2023-07-22T12:33:41,279 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=855ade4dc43fb86bf7ef486829b4203f, ASSIGN 2023-07-22T12:33:41,280 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=855ade4dc43fb86bf7ef486829b4203f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:41,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=855ade4dc43fb86bf7ef486829b4203f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:41,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 855ade4dc43fb86bf7ef486829b4203f, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:41,596 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,597 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:33:41,597 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,42129,1690029216993.regiongroup-1 2023-07-22T12:33:41,600 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1, suffix=, logDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993, archiveDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs, maxLogs=32 2023-07-22T12:33:41,612 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029221601, exclude list is [], retry=0 2023-07-22T12:33:41,616 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:33:41,619 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029221601 2023-07-22T12:33:41,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:33:41,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 855ade4dc43fb86bf7ef486829b4203f, NAME => 'hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:41,621 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:41,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:41,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,623 INFO [StoreOpener-855ade4dc43fb86bf7ef486829b4203f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,625 INFO [StoreOpener-855ade4dc43fb86bf7ef486829b4203f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 855ade4dc43fb86bf7ef486829b4203f columnFamilyName info 2023-07-22T12:33:41,625 DEBUG [StoreOpener-855ade4dc43fb86bf7ef486829b4203f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:41,626 INFO [StoreOpener-855ade4dc43fb86bf7ef486829b4203f-1 {}] regionserver.HStore(324): Store=855ade4dc43fb86bf7ef486829b4203f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:41,628 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,628 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:33:41,635 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:41,636 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 855ade4dc43fb86bf7ef486829b4203f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11433545120, jitterRate=0.06483186781406403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:41,637 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 855ade4dc43fb86bf7ef486829b4203f: 2023-07-22T12:33:41,639 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f., pid=6, masterSystemTime=1690029221590 2023-07-22T12:33:41,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,642 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:33:41,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=855ade4dc43fb86bf7ef486829b4203f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:41,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:33:41,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 855ade4dc43fb86bf7ef486829b4203f, server=jenkins-hbase3.apache.org,42129,1690029216993 in 210 msec 2023-07-22T12:33:41,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:33:41,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=855ade4dc43fb86bf7ef486829b4203f, ASSIGN in 374 msec 2023-07-22T12:33:41,654 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:41,655 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029221654"}]},"ts":"1690029221654"} 2023-07-22T12:33:41,657 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:33:41,660 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:41,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2990 sec 2023-07-22T12:33:41,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-07-22T12:33:41,676 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:41,676 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:41,676 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:33:41,722 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:33:41,737 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:41,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 27 msec 2023-07-22T12:33:41,747 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:33:41,759 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:41,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 17 msec 2023-07-22T12:33:41,773 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-07-22T12:33:41,776 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-07-22T12:33:41,776 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.671sec 2023-07-22T12:33:41,779 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:33:41,780 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:33:41,782 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:33:41,782 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:33:41,783 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:33:41,784 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44705,1690029213996-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:33:41,785 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44705,1690029213996-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:33:41,798 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:33:41,799 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:33:41,820 DEBUG [Listener at localhost/43917 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7179af93 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:33:41,824 DEBUG [Listener at localhost/43917 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49425891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:41,829 DEBUG [Listener at localhost/43917 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-07-22T12:33:41,829 DEBUG [Listener at localhost/43917 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-07-22T12:33:41,844 DEBUG [hconnection-0x49bbb496-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:41,855 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33304, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:41,865 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:33:41,865 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:33:41,865 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/test.cache.data in system properties and HBase conf 2023-07-22T12:33:41,865 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:33:41,866 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:33:41,867 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:33:41,867 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:33:41,867 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:33:41,867 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:33:41,867 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:33:41,867 INFO [Listener at localhost/43917 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:33:42,911 WARN [Thread-251 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:33:43,266 INFO [Thread-251 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:33:43,272 INFO [Thread-251 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41829_jobhistory____uj3zjg/webapp 2023-07-22T12:33:43,441 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:33:43,441 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:33:43,441 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:33:43,445 INFO [Thread-251 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:33:43,539 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:43,664 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:43,932 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:33:43,961 INFO [Thread-251 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41829 2023-07-22T12:33:44,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:33:44,162 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-07-22T12:33:44,170 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:33:44,170 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-07-22T12:33:44,172 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:44,172 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-07-22T12:33:44,172 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:33:44,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-07-22T12:33:44,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:33:44,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-07-22T12:33:44,405 WARN [Listener at jenkins-hbase3.apache.org/39669 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:33:44,483 WARN [Listener at jenkins-hbase3.apache.org/39669 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:33:44,483 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:33:44,489 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45523_cluster____yxpasu/webapp 2023-07-22T12:33:44,557 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:33:44,558 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:33:44,558 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:33:44,558 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:33:44,566 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:44,742 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:44,805 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:33:44,807 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-07-22T12:33:44,823 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:44,829 INFO [Listener at jenkins-hbase3.apache.org/39669 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45523 2023-07-22T12:33:45,219 WARN [Listener at jenkins-hbase3.apache.org/39805 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:33:45,242 WARN [Listener at jenkins-hbase3.apache.org/39805 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:33:45,242 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:33:45,248 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45711_node____sxhh91/webapp 2023-07-22T12:33:45,300 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:33:45,300 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:33:45,300 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:33:45,300 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:33:45,308 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:45,330 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:45,415 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:45,424 INFO [Listener at jenkins-hbase3.apache.org/39805 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45711 2023-07-22T12:33:45,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:33:45,566 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:33:45,574 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:33:45,575 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:33:45,597 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42287_node____.uzfywe/webapp 2023-07-22T12:33:45,689 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:33:45,689 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:33:45,689 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:33:45,690 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:33:45,700 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:45,724 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:45,827 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:33:45,837 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42287 2023-07-22T12:33:45,869 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:33:45,870 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:33927/backupUT 2023-07-22T12:33:45,882 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:45,887 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53054, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:45,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-07-22T12:33:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-07-22T12:33:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:33:45,927 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:45,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 32 msec 2023-07-22T12:33:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:33:46,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-07-22T12:33:46,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-07-22T12:33:46,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:33:46,047 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:46,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 25 msec 2023-07-22T12:33:46,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:33:46,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-07-22T12:33:46,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-07-22T12:33:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:33:46,159 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:46,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 20 msec 2023-07-22T12:33:46,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:33:46,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-07-22T12:33:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-07-22T12:33:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:33:46,269 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:46,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-07-22T12:33:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:33:46,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1690029225870', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1690029225870 2023-07-22T12:33:46,382 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1690029225870 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:46,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1690029225870" procId is: 13 2023-07-22T12:33:46,383 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:46,384 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1690029225870 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:33:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:33:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:33:46,820 INFO [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 36c0073c74700eeabcb5149b86d5da47, NAME => 'test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690029225870', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:33:47,239 DEBUG [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:47,240 DEBUG [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1668): Closing 36c0073c74700eeabcb5149b86d5da47, disabling compactions & flushes 2023-07-22T12:33:47,240 INFO [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,240 DEBUG [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,240 DEBUG [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. after waiting 0 ms 2023-07-22T12:33:47,240 DEBUG [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,240 INFO [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1909): Closed test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,240 DEBUG [RegionOpenAndInit-test-1690029225870-pool-0 {}] regionserver.HRegion(1622): Region close journal for 36c0073c74700eeabcb5149b86d5da47: 2023-07-22T12:33:47,241 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1690029225870 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:47,242 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690029227241"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029227241"}]},"ts":"1690029227241"} 2023-07-22T12:33:47,244 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:47,245 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1690029225870 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:47,246 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029225870","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029227245"}]},"ts":"1690029227245"} 2023-07-22T12:33:47,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029225870, state=ENABLING in hbase:meta 2023-07-22T12:33:47,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, ASSIGN}] 2023-07-22T12:33:47,254 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, ASSIGN 2023-07-22T12:33:47,256 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:47,407 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=36c0073c74700eeabcb5149b86d5da47, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:47,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 36c0073c74700eeabcb5149b86d5da47, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:33:47,571 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,571 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 36c0073c74700eeabcb5149b86d5da47, NAME => 'test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:47,571 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:47,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029225870 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:47,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,574 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,577 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36c0073c74700eeabcb5149b86d5da47 columnFamilyName f 2023-07-22T12:33:47,577 DEBUG [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:47,578 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.HStore(324): Store=36c0073c74700eeabcb5149b86d5da47/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:47,579 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,584 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:47,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:47,590 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 36c0073c74700eeabcb5149b86d5da47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11009720000, jitterRate=0.025360077619552612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:47,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 36c0073c74700eeabcb5149b86d5da47: 2023-07-22T12:33:47,592 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47., pid=15, masterSystemTime=1690029227563 2023-07-22T12:33:47,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:47,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=36c0073c74700eeabcb5149b86d5da47, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:47,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-07-22T12:33:47,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 36c0073c74700eeabcb5149b86d5da47, server=jenkins-hbase3.apache.org,42129,1690029216993 in 189 msec 2023-07-22T12:33:47,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-07-22T12:33:47,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, ASSIGN in 349 msec 2023-07-22T12:33:47,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1690029225870 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:47,605 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690029225870","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029227605"}]},"ts":"1690029227605"} 2023-07-22T12:33:47,607 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690029225870, state=ENABLED in hbase:meta 2023-07-22T12:33:47,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1690029225870 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:47,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1690029225870 in 1.2330 sec 2023-07-22T12:33:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:33:48,500 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1690029225870, procId: 13 completed 2023-07-22T12:33:48,501 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1bd11fa9 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:33:48,506 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b655551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:48,508 DEBUG [hconnection-0x567a83a5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:48,511 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33310, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:48,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42129 {}] regionserver.HRegion(8210): writing data to region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:33:48,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16900292258701', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16900292258701 2023-07-22T12:33:48,612 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900292258701 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:48,612 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:48,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16900292258701" procId is: 16 2023-07-22T12:33:48,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16900292258701 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:33:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:33:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:33:49,031 INFO [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 21e7da9035e8749de782db2e8bf7e35d, NAME => 'ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900292258701', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:49,048 DEBUG [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:49,048 DEBUG [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1668): Closing 21e7da9035e8749de782db2e8bf7e35d, disabling compactions & flushes 2023-07-22T12:33:49,048 INFO [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,048 DEBUG [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,048 DEBUG [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. after waiting 0 ms 2023-07-22T12:33:49,048 DEBUG [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,048 INFO [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,048 DEBUG [RegionOpenAndInit-ns2:test-16900292258701-pool-0 {}] regionserver.HRegion(1622): Region close journal for 21e7da9035e8749de782db2e8bf7e35d: 2023-07-22T12:33:49,050 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16900292258701 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:49,050 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029229050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029229050"}]},"ts":"1690029229050"} 2023-07-22T12:33:49,053 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:49,054 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16900292258701 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:49,054 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900292258701","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029229054"}]},"ts":"1690029229054"} 2023-07-22T12:33:49,056 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900292258701, state=ENABLING in hbase:meta 2023-07-22T12:33:49,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900292258701, region=21e7da9035e8749de782db2e8bf7e35d, ASSIGN}] 2023-07-22T12:33:49,061 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900292258701, region=21e7da9035e8749de782db2e8bf7e35d, ASSIGN 2023-07-22T12:33:49,062 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900292258701, region=21e7da9035e8749de782db2e8bf7e35d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:49,214 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=21e7da9035e8749de782db2e8bf7e35d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:49,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 21e7da9035e8749de782db2e8bf7e35d, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:33:49,374 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,374 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 21e7da9035e8749de782db2e8bf7e35d, NAME => 'ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:49,374 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:49,374 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900292258701 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,374 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:49,374 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,374 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,377 INFO [StoreOpener-21e7da9035e8749de782db2e8bf7e35d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,379 INFO [StoreOpener-21e7da9035e8749de782db2e8bf7e35d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21e7da9035e8749de782db2e8bf7e35d columnFamilyName f 2023-07-22T12:33:49,379 DEBUG [StoreOpener-21e7da9035e8749de782db2e8bf7e35d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:49,380 INFO [StoreOpener-21e7da9035e8749de782db2e8bf7e35d-1 {}] regionserver.HStore(324): Store=21e7da9035e8749de782db2e8bf7e35d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:49,381 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,382 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,385 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:33:49,392 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:49,393 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 21e7da9035e8749de782db2e8bf7e35d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11097857920, jitterRate=0.03356856107711792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:49,394 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 21e7da9035e8749de782db2e8bf7e35d: 2023-07-22T12:33:49,395 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d., pid=18, masterSystemTime=1690029229369 2023-07-22T12:33:49,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,397 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:33:49,398 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=21e7da9035e8749de782db2e8bf7e35d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:49,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-07-22T12:33:49,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 21e7da9035e8749de782db2e8bf7e35d, server=jenkins-hbase3.apache.org,42129,1690029216993 in 184 msec 2023-07-22T12:33:49,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-07-22T12:33:49,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900292258701, region=21e7da9035e8749de782db2e8bf7e35d, ASSIGN in 343 msec 2023-07-22T12:33:49,406 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16900292258701 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:49,407 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900292258701","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029229406"}]},"ts":"1690029229406"} 2023-07-22T12:33:49,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900292258701, state=ENABLED in hbase:meta 2023-07-22T12:33:49,412 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900292258701 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:49,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16900292258701 in 803 msec 2023-07-22T12:33:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:33:49,721 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16900292258701, procId: 16 completed 2023-07-22T12:33:49,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42129 {}] regionserver.HRegion(8210): writing data to region ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:33:49,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16900292258702', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16900292258702 2023-07-22T12:33:49,816 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900292258702 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:49,816 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:49,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16900292258702" procId is: 19 2023-07-22T12:33:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:33:49,817 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16900292258702 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:49,834 INFO [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9321bfacf9c9df682580ca2a0b229466, NAME => 'ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900292258702', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:33:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:33:50,247 DEBUG [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:50,247 DEBUG [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1668): Closing 9321bfacf9c9df682580ca2a0b229466, disabling compactions & flushes 2023-07-22T12:33:50,247 INFO [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,247 DEBUG [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,248 DEBUG [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. after waiting 0 ms 2023-07-22T12:33:50,248 DEBUG [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,248 INFO [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,248 DEBUG [RegionOpenAndInit-ns3:test-16900292258702-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9321bfacf9c9df682580ca2a0b229466: 2023-07-22T12:33:50,249 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16900292258702 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:50,250 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029230249"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029230249"}]},"ts":"1690029230249"} 2023-07-22T12:33:50,252 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:50,253 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16900292258702 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:50,253 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900292258702","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029230253"}]},"ts":"1690029230253"} 2023-07-22T12:33:50,255 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900292258702, state=ENABLING in hbase:meta 2023-07-22T12:33:50,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900292258702, region=9321bfacf9c9df682580ca2a0b229466, ASSIGN}] 2023-07-22T12:33:50,262 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900292258702, region=9321bfacf9c9df682580ca2a0b229466, ASSIGN 2023-07-22T12:33:50,263 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900292258702, region=9321bfacf9c9df682580ca2a0b229466, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:50,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=9321bfacf9c9df682580ca2a0b229466, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:50,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 9321bfacf9c9df682580ca2a0b229466, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:33:50,573 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 9321bfacf9c9df682580ca2a0b229466, NAME => 'ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:50,574 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:50,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900292258702 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:50,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,576 INFO [StoreOpener-9321bfacf9c9df682580ca2a0b229466-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,578 INFO [StoreOpener-9321bfacf9c9df682580ca2a0b229466-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9321bfacf9c9df682580ca2a0b229466 columnFamilyName f 2023-07-22T12:33:50,578 DEBUG [StoreOpener-9321bfacf9c9df682580ca2a0b229466-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:50,579 INFO [StoreOpener-9321bfacf9c9df682580ca2a0b229466-1 {}] regionserver.HStore(324): Store=9321bfacf9c9df682580ca2a0b229466/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:50,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns3/test-16900292258702/9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,580 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns3/test-16900292258702/9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,583 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:33:50,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns3/test-16900292258702/9321bfacf9c9df682580ca2a0b229466/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:50,587 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 9321bfacf9c9df682580ca2a0b229466; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11117461440, jitterRate=0.03539428114891052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:50,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 9321bfacf9c9df682580ca2a0b229466: 2023-07-22T12:33:50,614 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466., pid=21, masterSystemTime=1690029230569 2023-07-22T12:33:50,616 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,617 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:33:50,617 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=9321bfacf9c9df682580ca2a0b229466, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:50,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-07-22T12:33:50,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 9321bfacf9c9df682580ca2a0b229466, server=jenkins-hbase3.apache.org,42129,1690029216993 in 204 msec 2023-07-22T12:33:50,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-07-22T12:33:50,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900292258702, region=9321bfacf9c9df682580ca2a0b229466, ASSIGN in 363 msec 2023-07-22T12:33:50,625 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16900292258702 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:50,626 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900292258702","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029230625"}]},"ts":"1690029230625"} 2023-07-22T12:33:50,627 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900292258702, state=ENABLED in hbase:meta 2023-07-22T12:33:50,630 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900292258702 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:50,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16900292258702 in 818 msec 2023-07-22T12:33:50,808 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900292258702' 2023-07-22T12:33:50,809 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690029225870' 2023-07-22T12:33:50,809 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16900292258701' 2023-07-22T12:33:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:33:50,924 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16900292258702, procId: 19 completed 2023-07-22T12:33:50,924 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16900292258702 get assigned. Timeout = 60000ms 2023-07-22T12:33:50,926 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:33:50,931 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16900292258702 assigned to meta. Checking AM states. 2023-07-22T12:33:50,932 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:33:50,932 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16900292258702 assigned. 2023-07-22T12:33:50,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16900292258703', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16900292258703 2023-07-22T12:33:50,937 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900292258703 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:50,937 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:50,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16900292258703" procId is: 22 2023-07-22T12:33:50,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16900292258703 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:33:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:33:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:33:51,352 INFO [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9a6d02904febf9e4645bb4f3bf3e2671, NAME => 'ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16900292258703', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:33:51,765 DEBUG [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:51,765 DEBUG [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1668): Closing 9a6d02904febf9e4645bb4f3bf3e2671, disabling compactions & flushes 2023-07-22T12:33:51,765 INFO [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:51,765 DEBUG [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:51,765 DEBUG [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. after waiting 0 ms 2023-07-22T12:33:51,765 DEBUG [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:51,765 INFO [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:51,765 DEBUG [RegionOpenAndInit-ns4:test-16900292258703-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9a6d02904febf9e4645bb4f3bf3e2671: 2023-07-22T12:33:51,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16900292258703 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:51,767 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690029231767"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029231767"}]},"ts":"1690029231767"} 2023-07-22T12:33:51,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:51,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16900292258703 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:51,770 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900292258703","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029231770"}]},"ts":"1690029231770"} 2023-07-22T12:33:51,772 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900292258703, state=ENABLING in hbase:meta 2023-07-22T12:33:51,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900292258703, region=9a6d02904febf9e4645bb4f3bf3e2671, ASSIGN}] 2023-07-22T12:33:51,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900292258703, region=9a6d02904febf9e4645bb4f3bf3e2671, ASSIGN 2023-07-22T12:33:51,780 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16900292258703, region=9a6d02904febf9e4645bb4f3bf3e2671, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:51,931 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=9a6d02904febf9e4645bb4f3bf3e2671, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:51,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 9a6d02904febf9e4645bb4f3bf3e2671, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:33:52,091 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:52,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 9a6d02904febf9e4645bb4f3bf3e2671, NAME => 'ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:52,091 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:52,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900292258703 9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:52,092 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,092 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,093 INFO [StoreOpener-9a6d02904febf9e4645bb4f3bf3e2671-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,095 INFO [StoreOpener-9a6d02904febf9e4645bb4f3bf3e2671-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6d02904febf9e4645bb4f3bf3e2671 columnFamilyName f 2023-07-22T12:33:52,095 DEBUG [StoreOpener-9a6d02904febf9e4645bb4f3bf3e2671-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:52,096 INFO [StoreOpener-9a6d02904febf9e4645bb4f3bf3e2671-1 {}] regionserver.HStore(324): Store=9a6d02904febf9e4645bb4f3bf3e2671/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:52,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns4/test-16900292258703/9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns4/test-16900292258703/9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,100 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:33:52,102 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns4/test-16900292258703/9a6d02904febf9e4645bb4f3bf3e2671/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:52,103 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 9a6d02904febf9e4645bb4f3bf3e2671; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10492164960, jitterRate=-0.022840991616249084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:33:52,104 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 9a6d02904febf9e4645bb4f3bf3e2671: 2023-07-22T12:33:52,105 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671., pid=24, masterSystemTime=1690029232086 2023-07-22T12:33:52,107 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:52,107 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:33:52,108 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=9a6d02904febf9e4645bb4f3bf3e2671, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:52,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-07-22T12:33:52,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 9a6d02904febf9e4645bb4f3bf3e2671, server=jenkins-hbase3.apache.org,42129,1690029216993 in 176 msec 2023-07-22T12:33:52,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-07-22T12:33:52,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16900292258703, region=9a6d02904febf9e4645bb4f3bf3e2671, ASSIGN in 335 msec 2023-07-22T12:33:52,115 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16900292258703 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:52,116 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900292258703","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029232115"}]},"ts":"1690029232115"} 2023-07-22T12:33:52,117 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900292258703, state=ENABLED in hbase:meta 2023-07-22T12:33:52,119 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900292258703 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:52,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16900292258703 in 1.1860 sec 2023-07-22T12:33:53,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:33:53,047 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16900292258703, procId: 22 completed 2023-07-22T12:33:53,047 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16900292258703 get assigned. Timeout = 60000ms 2023-07-22T12:33:53,048 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:33:53,051 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16900292258703 assigned to meta. Checking AM states. 2023-07-22T12:33:53,052 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:33:53,052 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16900292258703 assigned. 2023-07-22T12:33:53,052 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1bd11fa9 to 127.0.0.1:56341 2023-07-22T12:33:53,052 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:53,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:33:53,104 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=585, OpenFileDescriptor=667, MaxFileDescriptor=60000, SystemLoadAverage=124, ProcessCount=169, AvailableMemoryMB=16409 2023-07-22T12:33:53,104 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2023-07-22T12:33:53,108 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(82): create full backup image for all tables 2023-07-22T12:33:53,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$13(2699): Client=jenkins//172.31.12.81 modify table test-1690029225870 from 'test-1690029225870', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1690029225870', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=test-1690029225870 2023-07-22T12:33:53,168 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/.tabledesc/.tableinfo.0000000001.367 2023-07-22T12:33:53,169 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/.tabledesc/.tableinfo.0000000002.598 2023-07-22T12:33:53,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=test-1690029225870}] 2023-07-22T12:33:53,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, REOPEN/MOVE}] 2023-07-22T12:33:53,190 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, REOPEN/MOVE 2023-07-22T12:33:53,191 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=36c0073c74700eeabcb5149b86d5da47, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:53,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; CloseRegionProcedure 36c0073c74700eeabcb5149b86d5da47, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:53,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(119): Close 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:53,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1668): Closing 36c0073c74700eeabcb5149b86d5da47, disabling compactions & flushes 2023-07-22T12:33:53,357 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1690): Closing region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:53,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:53,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1778): Acquired close lock on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. after waiting 0 ms 2023-07-22T12:33:53,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1788): Updates disabled for region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:53,358 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2819): Flushing 36c0073c74700eeabcb5149b86d5da47 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:33:53,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f/1c5b4c1fb8df4bc88bb1e0637f47f565 is 37, key is row10/f:q1/1690029228526/Put/seqid=0 2023-07-22T12:33:53,857 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f/1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:33:53,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:33:53,925 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f/1c5b4c1fb8df4bc88bb1e0637f47f565 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:33:53,933 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:33:53,937 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 36c0073c74700eeabcb5149b86d5da47 in 579ms, sequenceid=103, compaction requested=false 2023-07-22T12:33:53,941 DEBUG [StoreFileCloser-36c0073c74700eeabcb5149b86d5da47-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:33:53,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:33:53,947 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:33:53,948 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1909): Closed test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:53,948 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1622): Region close journal for 36c0073c74700eeabcb5149b86d5da47: 2023-07-22T12:33:53,949 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionServer(3731): Not adding moved region record: 36c0073c74700eeabcb5149b86d5da47 to self. 2023-07-22T12:33:53,951 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(163): Closed 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:53,952 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=36c0073c74700eeabcb5149b86d5da47, regionState=CLOSED 2023-07-22T12:33:53,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-07-22T12:33:53,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; CloseRegionProcedure 36c0073c74700eeabcb5149b86d5da47, server=jenkins-hbase3.apache.org,42129,1690029216993 in 760 msec 2023-07-22T12:33:53,957 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=true 2023-07-22T12:33:54,107 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=36c0073c74700eeabcb5149b86d5da47, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:54,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=29, ppid=27, state=RUNNABLE; OpenRegionProcedure 36c0073c74700eeabcb5149b86d5da47, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:54,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900292258702 2023-07-22T12:33:54,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900292258702 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:33:54,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900292258703 2023-07-22T12:33:54,161 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900292258703 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:33:54,162 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900292258701 2023-07-22T12:33:54,162 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900292258701 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:33:54,267 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:54,267 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 36c0073c74700eeabcb5149b86d5da47, NAME => 'test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:54,268 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:54,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690029225870 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:54,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,268 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,270 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,272 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36c0073c74700eeabcb5149b86d5da47 columnFamilyName f 2023-07-22T12:33:54,272 DEBUG [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:54,282 DEBUG [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:33:54,282 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.HStore(324): Store=36c0073c74700eeabcb5149b86d5da47/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:54,282 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,284 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36c0073c74700eeabcb5149b86d5da47 columnFamilyName f3 2023-07-22T12:33:54,284 DEBUG [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:54,285 INFO [StoreOpener-36c0073c74700eeabcb5149b86d5da47-1 {}] regionserver.HStore(324): Store=36c0073c74700eeabcb5149b86d5da47/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:54,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,287 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,289 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1690029225870 descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:33:54,291 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:33:54,292 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 36c0073c74700eeabcb5149b86d5da47; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9811865440, jitterRate=-0.0861988216638565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:33:54,292 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 36c0073c74700eeabcb5149b86d5da47: 2023-07-22T12:33:54,293 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47., pid=29, masterSystemTime=1690029234262 2023-07-22T12:33:54,295 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:54,295 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:33:54,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=36c0073c74700eeabcb5149b86d5da47, regionState=OPEN, openSeqNum=107, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:54,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=29, resume processing ppid=27 2023-07-22T12:33:54,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, ppid=27, state=SUCCESS; OpenRegionProcedure 36c0073c74700eeabcb5149b86d5da47, server=jenkins-hbase3.apache.org,42129,1690029216993 in 188 msec 2023-07-22T12:33:54,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-07-22T12:33:54,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=test-1690029225870, region=36c0073c74700eeabcb5149b86d5da47, REOPEN/MOVE in 1.1110 sec 2023-07-22T12:33:54,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=26, resume processing ppid=25 2023-07-22T12:33:54,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, ppid=25, state=SUCCESS; ReopenTableRegionsProcedure table=test-1690029225870 in 1.1320 sec 2023-07-22T12:33:54,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; ModifyTableProcedure table=test-1690029225870 in 1.1850 sec 2023-07-22T12:33:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:33:54,320 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x510eb700 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:33:54,326 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2467e397, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:54,328 DEBUG [hconnection-0x44cd13d2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:54,330 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35054, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:54,363 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:33:54,365 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57938, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:33:54,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-07-22T12:33:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=30, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-07-22T12:33:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=30 2023-07-22T12:33:54,392 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:33:54,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 20 msec 2023-07-22T12:33:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=30 2023-07-22T12:33:54,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-07-22T12:33:54,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:54,503 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:54,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 31 2023-07-22T12:33:54,504 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-07-22T12:33:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-07-22T12:33:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-07-22T12:33:54,917 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-07-22T12:33:55,330 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:55,330 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 8e2fda6a14d55f9b3ace42576496dba7, disabling compactions & flushes 2023-07-22T12:33:55,330 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,330 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,330 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. after waiting 0 ms 2023-07-22T12:33:55,330 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,330 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,330 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:33:55,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:55,332 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029235331"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029235331"}]},"ts":"1690029235331"} 2023-07-22T12:33:55,334 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:55,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:55,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029235335"}]},"ts":"1690029235335"} 2023-07-22T12:33:55,336 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:33:55,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN}] 2023-07-22T12:33:55,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN 2023-07-22T12:33:55,344 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:55,496 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:55,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-07-22T12:33:55,655 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(130): Open backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:55,656 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:55,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(888): Instantiated backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:55,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7296): checking encryption for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,656 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7299): checking classloading for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,658 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,660 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName meta 2023-07-22T12:33:55,660 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:55,662 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:55,662 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,663 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName session 2023-07-22T12:33:55,663 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:55,664 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:55,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,666 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,667 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:33:55,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1076): writing seq id for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:55,672 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:55,673 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): Opened 8e2fda6a14d55f9b3ace42576496dba7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12032390400, jitterRate=0.12060368061065674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:33:55,673 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(992): Region open journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:33:55,674 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., pid=33, masterSystemTime=1690029235651 2023-07-22T12:33:55,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,676 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(158): Opened backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:55,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:55,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-07-22T12:33:55,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 180 msec 2023-07-22T12:33:55,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=32, resume processing ppid=31 2023-07-22T12:33:55,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, ppid=31, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN in 339 msec 2023-07-22T12:33:55,684 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:55,684 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029235684"}]},"ts":"1690029235684"} 2023-07-22T12:33:55,686 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:33:55,688 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:55,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, state=SUCCESS; CreateTableProcedure table=backup:system in 1.1900 sec 2023-07-22T12:33:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-07-22T12:33:56,612 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 31 completed 2023-07-22T12:33:56,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:33:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-07-22T12:33:56,630 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:33:56,630 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:56,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 34 2023-07-22T12:33:56,631 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:33:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:33:56,645 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e50e27a27a191e84f289cbbf597a7b51, NAME => 'backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:33:56,658 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:56,658 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing e50e27a27a191e84f289cbbf597a7b51, disabling compactions & flushes 2023-07-22T12:33:56,658 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:56,658 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:56,658 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. after waiting 0 ms 2023-07-22T12:33:56,658 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:56,658 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:56,658 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for e50e27a27a191e84f289cbbf597a7b51: 2023-07-22T12:33:56,659 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:33:56,660 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690029236659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029236659"}]},"ts":"1690029236659"} 2023-07-22T12:33:56,661 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:33:56,662 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:33:56,662 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029236662"}]},"ts":"1690029236662"} 2023-07-22T12:33:56,664 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-07-22T12:33:56,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=e50e27a27a191e84f289cbbf597a7b51, ASSIGN}] 2023-07-22T12:33:56,670 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=e50e27a27a191e84f289cbbf597a7b51, ASSIGN 2023-07-22T12:33:56,671 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=e50e27a27a191e84f289cbbf597a7b51, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,42129,1690029216993; forceNewPlan=false, retain=false 2023-07-22T12:33:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:33:56,811 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16900292258703' 2023-07-22T12:33:56,811 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:33:56,822 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=e50e27a27a191e84f289cbbf597a7b51, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:56,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure e50e27a27a191e84f289cbbf597a7b51, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:33:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:33:56,981 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:56,981 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7254): Opening region: {ENCODED => e50e27a27a191e84f289cbbf597a7b51, NAME => 'backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:33:56,981 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:33:56,981 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:33:56,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7296): checking encryption for e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7299): checking classloading for e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,983 INFO [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,985 INFO [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50e27a27a191e84f289cbbf597a7b51 columnFamilyName meta 2023-07-22T12:33:56,985 DEBUG [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:56,986 INFO [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] regionserver.HStore(324): Store=e50e27a27a191e84f289cbbf597a7b51/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:56,986 INFO [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,988 INFO [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50e27a27a191e84f289cbbf597a7b51 columnFamilyName session 2023-07-22T12:33:56,988 DEBUG [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:33:56,989 INFO [StoreOpener-e50e27a27a191e84f289cbbf597a7b51-1 {}] regionserver.HStore(324): Store=e50e27a27a191e84f289cbbf597a7b51/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:33:56,990 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system_bulk/e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,990 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system_bulk/e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,992 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:33:56,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1076): writing seq id for e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:33:56,996 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system_bulk/e50e27a27a191e84f289cbbf597a7b51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:33:56,997 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): Opened e50e27a27a191e84f289cbbf597a7b51; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10156072160, jitterRate=-0.054142072796821594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:33:56,998 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(992): Region open journal for e50e27a27a191e84f289cbbf597a7b51: 2023-07-22T12:33:56,999 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51., pid=36, masterSystemTime=1690029236977 2023-07-22T12:33:57,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:57,001 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:33:57,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=e50e27a27a191e84f289cbbf597a7b51, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:33:57,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=36, resume processing ppid=35 2023-07-22T12:33:57,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure e50e27a27a191e84f289cbbf597a7b51, server=jenkins-hbase3.apache.org,42129,1690029216993 in 179 msec 2023-07-22T12:33:57,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-07-22T12:33:57,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=e50e27a27a191e84f289cbbf597a7b51, ASSIGN in 338 msec 2023-07-22T12:33:57,009 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:33:57,009 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029237009"}]},"ts":"1690029237009"} 2023-07-22T12:33:57,011 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-07-22T12:33:57,013 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:33:57,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 386 msec 2023-07-22T12:33:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:33:57,236 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 34 completed 2023-07-22T12:33:57,248 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:33:57,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029237297 (current time:1690029237297). 2023-07-22T12:33:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:33:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:33:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4b3a4829 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:33:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ece0da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:33:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:33:57,313 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35066, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:33:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4b3a4829 to 127.0.0.1:56341 2023-07-22T12:33:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:33:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:33:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:33:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-07-22T12:33:57,331 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:33:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:33:57,337 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:33:57,340 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:33:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:33:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:33:57,753 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:33:57,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7}] 2023-07-22T12:33:57,763 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:57,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=38 2023-07-22T12:33:57,923 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:57,925 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=45 B heapSize=632 B 2023-07-22T12:33:57,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:33:57,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/22c35e6d7bd14d1cad28782199ed075b is 49, key is activesession:/session:c/1690029237282/Put/seqid=0 2023-07-22T12:33:58,351 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:33:58,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:33:58,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/22c35e6d7bd14d1cad28782199ed075b as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:33:58,371 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, entries=1, sequenceid=5, filesize=4.9 K 2023-07-22T12:33:58,373 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 448ms, sequenceid=5, compaction requested=false 2023-07-22T12:33:58,374 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2520): Flush status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:33:58,374 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. for snapshot_backup_system completed. 2023-07-22T12:33:58,375 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:33:58,382 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:33:58,386 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:33:58,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b] hfiles 2023-07-22T12:33:58,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b for snapshot=snapshot_backup_system 2023-07-22T12:33:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:33:58,802 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:33:58,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=38 2023-07-22T12:33:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=38 2023-07-22T12:33:58,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:58,810 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:33:58,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=38, resume processing ppid=37 2023-07-22T12:33:58,813 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:33:58,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, ppid=37, state=SUCCESS; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 in 1.0550 sec 2023-07-22T12:33:58,815 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:33:58,818 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:33:58,818 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:33:58,821 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:33:59,244 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:33:59,257 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:33:59,258 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:33:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:33:59,451 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:00,087 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:00,087 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-07-22T12:34:00,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.7630 sec 2023-07-22T12:34:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:34:01,442 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 37 completed 2023-07-22T12:34:01,442 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(115): Backup backup_1690029234345 started at 1690029241442. 2023-07-22T12:34:01,466 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(119): Backup session backup_1690029234345 has been started. 2023-07-22T12:34:01,469 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-07-22T12:34:01,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:01,487 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:01,487 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:01,488 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:01,488 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:01,489 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,489 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,492 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:34:01,492 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,492 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:01,492 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:01,492 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,492 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:01,492 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,493 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:01,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,493 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:01,495 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:01,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:01,496 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:01,496 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:01,497 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:01,497 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:01,497 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:01,500 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,500 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:01,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:01,500 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,500 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:01,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:01,503 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:01,503 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@631cb4a0[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:01,504 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:01,504 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,507 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,507 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,507 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:01,507 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,507 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:01,508 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:01,508 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,509 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029219003 highest: 1690029221601 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,509 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1:(num 1690029221601) roll requested 2023-07-22T12:34:01,521 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509, exclude list is [], retry=0 2023-07-22T12:34:01,526 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:01,530 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029221601 with entries=11, filesize=2.04 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:01,531 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:01,531 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029221601 is not closed yet, will try archiving it next time 2023-07-22T12:34:01,532 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0:(num 1690029219003) roll requested 2023-07-22T12:34:01,544 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532, exclude list is [], retry=0 2023-07-22T12:34:01,548 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:01,550 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 with entries=18, filesize=3.96 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:34:01,551 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:01,551 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 is not closed yet, will try archiving it next time 2023-07-22T12:34:01,551 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta:.meta(num 1690029219975) roll requested 2023-07-22T12:34:01,563 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta, exclude list is [], retry=0 2023-07-22T12:34:01,566 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:01,570 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta with entries=40, filesize=12.94 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta 2023-07-22T12:34:01,570 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:01,570 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:01,590 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 81 2023-07-22T12:34:01,590 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029241532 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,591 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:01,593 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:54064, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:34:01,610 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:01,610 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,42129,1690029216993' in zk 2023-07-22T12:34:01,613 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,613 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:01,613 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:01,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,613 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:01,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:01,613 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:01,614 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:01,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:01,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:01,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,42129,1690029216993': 2023-07-22T12:34:01,618 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:01,618 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:01,618 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:01,618 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:01,618 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:01,618 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:01,622 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,622 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,623 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:34:01,623 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:01,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:01,623 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:01,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:01,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:01,624 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:01,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,626 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,626 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,626 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:01,626 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:01,627 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): regionserver:42129-0x1000060a6040001, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-07-22T12:34:01,636 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:01,636 DEBUG [Listener at localhost/43917-EventThread {}] zookeeper.ZKWatcher(604): master:44705-0x1000060a6040000, quorum=127.0.0.1:56341, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:01,637 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:01,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:01,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:01,638 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:01,638 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:01,640 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-07-22T12:34:01,640 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-07-22T12:34:01,740 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-07-22T12:34:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:01,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } 2023-07-22T12:34:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029241750 (current time:1690029241750). 2023-07-22T12:34:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029241749_default_test-1690029225870 VERSION not specified, setting to 2 2023-07-22T12:34:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x21e6e3f3 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:34:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db59487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:01,759 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42906, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x21e6e3f3 to 127.0.0.1:56341 2023-07-22T12:34:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:01,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } 2023-07-22T12:34:01,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-07-22T12:34:01,763 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:01,764 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:01,767 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:02,176 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:02,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 36c0073c74700eeabcb5149b86d5da47}] 2023-07-22T12:34:02,179 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:34:02,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2023-07-22T12:34:02,331 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:34:02,332 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2819): Flushing 36c0073c74700eeabcb5149b86d5da47 2/2 column families, dataSize=324 B heapSize=1.25 KB 2023-07-22T12:34:02,347 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f3/b7a678e43b8843f4a246072ebd55d154 is 58, key is row-test-1690029225870-3-0/f3:q1/1690029234334/Put/seqid=0 2023-07-22T12:34:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:02,757 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f3/b7a678e43b8843f4a246072ebd55d154 2023-07-22T12:34:02,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b7a678e43b8843f4a246072ebd55d154 2023-07-22T12:34:02,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f3/b7a678e43b8843f4a246072ebd55d154 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f3/b7a678e43b8843f4a246072ebd55d154 2023-07-22T12:34:02,773 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f3/b7a678e43b8843f4a246072ebd55d154, entries=6, sequenceid=115, filesize=5.2 K 2023-07-22T12:34:02,775 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~1008 B/1008, currentSize=0 B/0 for 36c0073c74700eeabcb5149b86d5da47 in 443ms, sequenceid=115, compaction requested=false 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2520): Flush status journal for 36c0073c74700eeabcb5149b86d5da47: 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. for snapshot_1690029241749_default_test-1690029225870 completed. 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(240): Storing 'test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47.' region-info for snapshot=snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565] hfiles 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565 for snapshot=snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f3/b7a678e43b8843f4a246072ebd55d154] hfiles 2023-07-22T12:34:02,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f3/b7a678e43b8843f4a246072ebd55d154 for snapshot=snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:02,812 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-07-22T12:34:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:03,184 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:34:03,184 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=40 2023-07-22T12:34:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=40 2023-07-22T12:34:03,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029241749_default_test-1690029225870 on region 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:34:03,186 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:34:03,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-07-22T12:34:03,189 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:03,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; SnapshotRegionProcedure 36c0073c74700eeabcb5149b86d5da47 in 1.0110 sec 2023-07-22T12:34:03,190 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:03,190 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:03,191 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:03,191 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:03,606 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:03,614 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:03,615 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_1690029241749_default_test-1690029225870 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:04,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:34:04,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:34:04,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:34:04,161 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-07-22T12:34:04,162 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690029225870 2023-07-22T12:34:04,162 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690029225870 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:34:04,447 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:04,447 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-07-22T12:34:04,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1690029241749_default_test-1690029225870 table=test-1690029225870 type=FLUSH ttl=0 } in 2.6860 sec 2023-07-22T12:34:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:34:05,872 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1690029225870, procId: 39 completed 2023-07-22T12:34:05,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } 2023-07-22T12:34:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029245873 (current time:1690029245873). 2023-07-22T12:34:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690029245872_ns2_test-16900292258701 VERSION not specified, setting to 2 2023-07-22T12:34:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x79b928c0 to 127.0.0.1:56341 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:34:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fd6a57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:05,884 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x79b928c0 to 127.0.0.1:56341 2023-07-22T12:34:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } 2023-07-22T12:34:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-07-22T12:34:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:05,902 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:05,904 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:05,906 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:06,318 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:06,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 21e7da9035e8749de782db2e8bf7e35d}] 2023-07-22T12:34:06,320 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:34:06,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=42 2023-07-22T12:34:06,472 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:34:06,474 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2819): Flushing 21e7da9035e8749de782db2e8bf7e35d 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:34:06,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/.tmp/f/3b01f0ac8176452fa5d133e559bec9a6 is 37, key is row10/f:q1/1690029229736/Put/seqid=0 2023-07-22T12:34:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:06,901 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/.tmp/f/3b01f0ac8176452fa5d133e559bec9a6 2023-07-22T12:34:06,909 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3b01f0ac8176452fa5d133e559bec9a6 2023-07-22T12:34:06,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/.tmp/f/3b01f0ac8176452fa5d133e559bec9a6 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/3b01f0ac8176452fa5d133e559bec9a6 2023-07-22T12:34:06,917 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/3b01f0ac8176452fa5d133e559bec9a6, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:34:06,918 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 21e7da9035e8749de782db2e8bf7e35d in 444ms, sequenceid=103, compaction requested=false 2023-07-22T12:34:06,919 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2520): Flush status journal for 21e7da9035e8749de782db2e8bf7e35d: 2023-07-22T12:34:06,919 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. for snapshot_1690029245872_ns2_test-16900292258701 completed. 2023-07-22T12:34:06,919 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d.' region-info for snapshot=snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:06,920 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:06,920 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/3b01f0ac8176452fa5d133e559bec9a6] hfiles 2023-07-22T12:34:06,920 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/3b01f0ac8176452fa5d133e559bec9a6 for snapshot=snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:07,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:34:07,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=42 2023-07-22T12:34:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=42 2023-07-22T12:34:07,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690029245872_ns2_test-16900292258701 on region 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:34:07,328 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:34:07,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=42, resume processing ppid=41 2023-07-22T12:34:07,331 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:07,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, ppid=41, state=SUCCESS; SnapshotRegionProcedure 21e7da9035e8749de782db2e8bf7e35d in 1.0110 sec 2023-07-22T12:34:07,332 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:07,333 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:07,333 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:07,334 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:07,747 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:07,753 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:07,753 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_1690029245872_ns2_test-16900292258701 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:08,575 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:08,575 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-07-22T12:34:08,576 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1690029245872_ns2_test-16900292258701 table=ns2:test-16900292258701 type=FLUSH ttl=0 } in 2.6810 sec 2023-07-22T12:34:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-07-22T12:34:10,006 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns2:test-16900292258701, procId: 41 completed 2023-07-22T12:34:10,006 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1690029234345 2023-07-22T12:34:10,006 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-07-22T12:34:10,008 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-07-22T12:34:10,011 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029234345_test-1690029225870 2023-07-22T12:34:10,011 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029241749_default_test-1690029225870 to hdfs://localhost:33927/backupUT/backup_1690029234345/default/test-1690029225870/ 2023-07-22T12:34:10,020 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:10,050 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33927, inputRoot=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:34:10,050 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,44705,1690029213996_-352878067_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33927/backupUT/backup_1690029234345/default/test-1690029225870, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33927/backupUT/backup_1690029234345/default/test-1690029225870/.hbase-snapshot/.tmp/snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:10,050 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33927, inputRoot=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203, snapshotDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_1690029241749_default_test-1690029225870. 2023-07-22T12:34:10,063 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_1690029241749_default_test-1690029225870 to hdfs://localhost:33927/backupUT/backup_1690029234345/default/test-1690029225870/.hbase-snapshot/.tmp/snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:10,492 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:10,492 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:10,493 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:10,494 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:10,497 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:34:10,501 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36266, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:34:11,476 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-3682894179872826124.jar 2023-07-22T12:34:11,476 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,477 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,477 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,478 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,479 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,479 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,480 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,480 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,481 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:11,482 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:11,483 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:11,484 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:11,485 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:11,485 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:11,486 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:11,486 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:11,486 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:11,487 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:11,487 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:11,488 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:11,488 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:11,492 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:11,492 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:11,493 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:11,493 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:11,493 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:11,494 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:11,495 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:11,552 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:34:11,574 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029241749_default_test-1690029225870' hfile list 2023-07-22T12:34:11,584 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(674): export split=0 size=13.4 K 2023-07-22T12:34:11,780 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029251736/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:11,780 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029251736/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:11,954 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:34:12,012 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565 output=hdfs://localhost:33927/backupUT/backup_1690029234345/default/test-1690029225870/archive/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:34:12,012 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-07-22T12:34:12,448 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f3/b7a678e43b8843f4a246072ebd55d154 output=hdfs://localhost:33927/backupUT/backup_1690029234345/default/test-1690029225870/archive/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f3/b7a678e43b8843f4a246072ebd55d154 2023-07-22T12:34:12,448 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=5370 (5.2 K) time=0sec 2.561M/sec 2023-07-22T12:34:13,841 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:13,842 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:13,851 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:13,851 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029241749_default_test-1690029225870 finished. 2023-07-22T12:34:13,851 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1690029234345_ns2:test-16900292258701 2023-07-22T12:34:13,851 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1690029245872_ns2_test-16900292258701 to hdfs://localhost:33927/backupUT/backup_1690029234345/ns2/test-16900292258701/ 2023-07-22T12:34:13,852 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-07-22T12:34:13,875 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33927, inputRoot=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203 2023-07-22T12:34:13,875 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,44705,1690029213996_39280938_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33927/backupUT/backup_1690029234345/ns2/test-16900292258701, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33927/backupUT/backup_1690029234345/ns2/test-16900292258701/.hbase-snapshot/.tmp/snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:13,875 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33927, inputRoot=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203, snapshotDir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_1690029245872_ns2_test-16900292258701. 2023-07-22T12:34:13,884 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_1690029245872_ns2_test-16900292258701 to hdfs://localhost:33927/backupUT/backup_1690029234345/ns2/test-16900292258701/.hbase-snapshot/.tmp/snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:14,302 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:14,302 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:14,303 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:14,303 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,292 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-3590116902127407969.jar 2023-07-22T12:34:15,293 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,293 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,293 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,294 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,294 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,295 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,295 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,297 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,297 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:15,298 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:15,298 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:15,299 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:15,299 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:15,300 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:15,300 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:15,301 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:15,301 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:15,301 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:15,302 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:15,303 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:15,303 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:15,304 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:15,304 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:15,305 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:15,305 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:15,305 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-07-22T12:34:15,306 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:15,307 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:15,334 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-07-22T12:34:15,352 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1690029245872_ns2_test-16900292258701' hfile list 2023-07-22T12:34:15,355 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-07-22T12:34:15,429 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029255393/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:15,429 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029255393/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:15,494 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-07-22T12:34:15,530 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/3b01f0ac8176452fa5d133e559bec9a6 output=hdfs://localhost:33927/backupUT/backup_1690029234345/ns2/test-16900292258701/archive/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/3b01f0ac8176452fa5d133e559bec9a6 2023-07-22T12:34:15,530 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-07-22T12:34:16,472 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-07-22T12:34:16,474 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-07-22T12:34:16,483 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:16,484 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1690029245872_ns2_test-16900292258701 finished. 2023-07-22T12:34:16,484 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1014): test-1690029225870 2023-07-22T12:34:16,484 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1014): ns2:test-16900292258701 2023-07-22T12:34:16,583 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029234345 2023-07-22T12:34:16,583 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:34:16,997 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33927/backupUT/backup_1690029234345/.backup.manifest 2023-07-22T12:34:16,997 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1690029234345 2023-07-22T12:34:16,997 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-07-22T12:34:17,403 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33927/backupUT/backup_1690029234345/.backup.manifest 2023-07-22T12:34:17,403 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(358): Backup backup_1690029234345 finished: type=FULL,tablelist=test-1690029225870;ns2:test-16900292258701,targetRootDir=hdfs://localhost:33927/backupUT,startts=1690029241442,completets=1690029256579,bytescopied=0 2023-07-22T12:34:17,403 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:34:17,404 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:17,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029241749_default_test-1690029225870" 2023-07-22T12:34:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029241749_default_test-1690029225870 2023-07-22T12:34:17,415 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029241749_default_test-1690029225870 for backup backup_1690029234345 succeeded. 2023-07-22T12:34:17,415 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:17,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690029245872_ns2_test-16900292258701" 2023-07-22T12:34:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690029245872_ns2_test-16900292258701 2023-07-22T12:34:17,422 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1690029245872_ns2_test-16900292258701 for backup backup_1690029234345 succeeded. 2023-07-22T12:34:17,423 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:17,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:17,438 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:17,446 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:17,463 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(376): Backup backup_1690029234345 completed. 2023-07-22T12:34:17,584 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(104): writing 99 rows to test-1690029225870 2023-07-22T12:34:17,601 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(108): written 99 rows to test-1690029225870 2023-07-22T12:34:17,617 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(120): written 5 rows to ns2:test-16900292258701 2023-07-22T12:34:17,618 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 0 2023-07-22T12:34:17,654 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56d57503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:17,655 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:17,656 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36276, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:17,661 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:17,663 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33456, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:17,735 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:17,739 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:34:17,740 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(240): BackupId=backup_1690029257700,startts=0,failedts=1690029257739,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-07-22T12:34:17,740 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:34:17,742 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1530): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-07-22T12:34:17,742 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:17,744 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-07-22T12:34:17,744 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-07-22T12:34:17,745 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:17,747 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(246): Backup backup_1690029257700 failed. Backup session finished. Status: FAILURE 2023-07-22T12:34:17,748 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:17,748 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] ... 41 more 2023-07-22T12:34:17,752 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 1 2023-07-22T12:34:17,769 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22126739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:17,770 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:17,772 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:17,775 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:17,776 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:17,822 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:17,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029257825 (current time:1690029257825). 2023-07-22T12:34:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6206a390, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:17,840 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33478, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-07-22T12:34:17,845 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:17,846 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:17,848 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:18,256 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:18,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7}] 2023-07-22T12:34:18,258 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:18,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-07-22T12:34:18,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:18,410 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=1.83 KB heapSize=3.35 KB 2023-07-22T12:34:18,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 is 171, key is trslm:hdfs://localhost:33927/backupUT\x00ns2:test-16900292258701/meta:log-roll-map/1690029256546/Put/seqid=0 2023-07-22T12:34:18,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:18,829 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:18,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:18,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/9192f5afe4e84de9ade12479dc9882a5 is 459, key is session:backup_1690029234345/session:context/1690029257445/Put/seqid=0 2023-07-22T12:34:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:19,244 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.07 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:19,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:19,252 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:19,260 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0, entries=6, sequenceid=20, filesize=5.8 K 2023-07-22T12:34:19,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/9192f5afe4e84de9ade12479dc9882a5 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:19,266 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5, entries=3, sequenceid=20, filesize=5.7 K 2023-07-22T12:34:19,267 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.83 KB/1879, heapSize ~3.32 KB/3400, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 857ms, sequenceid=20, compaction requested=false 2023-07-22T12:34:19,267 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:19,267 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. for snapshot_backup_system completed. 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0] hfiles 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 for snapshot=snapshot_backup_system 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5] hfiles 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b for snapshot=snapshot_backup_system 2023-07-22T12:34:19,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 for snapshot=snapshot_backup_system 2023-07-22T12:34:19,674 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:19,674 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-07-22T12:34:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=44 2023-07-22T12:34:19,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:19,676 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:19,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=44, resume processing ppid=43 2023-07-22T12:34:19,678 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:19,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 in 1.4210 sec 2023-07-22T12:34:19,679 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:19,680 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:19,680 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:19,681 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:20,094 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:20,101 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:20,102 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:20,922 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:20,922 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-07-22T12:34:20,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0800 sec 2023-07-22T12:34:21,849 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] procedure2.ProcedureExecutor(1033): Stored pid=45, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE; InitializeStoreFileTrackerProcedure table=test-1690029225870 2023-07-22T12:34:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-07-22T12:34:21,952 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(115): Backup backup_1690029257800 started at 1690029261952. 2023-07-22T12:34:21,953 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(119): Backup session backup_1690029257800 has been started. 2023-07-22T12:34:21,954 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:34:21,954 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(240): BackupId=backup_1690029257800,startts=1690029261952,failedts=1690029261954,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-07-22T12:34:21,954 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:34:21,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:34:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:34:21,977 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029261977"}]},"ts":"1690029261977"} 2023-07-22T12:34:21,978 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:34:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:34:21,980 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:34:21,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN}] 2023-07-22T12:34:21,983 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN 2023-07-22T12:34:21,983 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:21,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:34:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:34:22,136 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(119): Close 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:22,137 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1668): Closing 8e2fda6a14d55f9b3ace42576496dba7, disabling compactions & flushes 2023-07-22T12:34:22,137 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1690): Closing region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:22,137 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:22,137 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. after waiting 0 ms 2023-07-22T12:34:22,137 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:22,137 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=214 B heapSize=800 B 2023-07-22T12:34:22,142 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/2a8fea4315254ed4ac54cbb5a9aa3953 is 218, key is session:backup_1690029257800/session:context/1690029261952/Put/seqid=0 2023-07-22T12:34:22,263 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/.tabledesc/.tableinfo.0000000002.598 2023-07-22T12:34:22,263 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/.tabledesc/.tableinfo.0000000003.640 2023-07-22T12:34:22,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, state=SUCCESS; InitializeStoreFileTrackerProcedure table=test-1690029225870 in 420 msec 2023-07-22T12:34:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:34:22,547 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/2a8fea4315254ed4ac54cbb5a9aa3953 2023-07-22T12:34:22,552 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2a8fea4315254ed4ac54cbb5a9aa3953 2023-07-22T12:34:22,553 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/2a8fea4315254ed4ac54cbb5a9aa3953 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/2a8fea4315254ed4ac54cbb5a9aa3953 2023-07-22T12:34:22,559 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/2a8fea4315254ed4ac54cbb5a9aa3953, entries=1, sequenceid=24, filesize=5.2 K 2023-07-22T12:34:22,560 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 423ms, sequenceid=24, compaction requested=true 2023-07-22T12:34:22,563 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:22,565 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:22,567 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:22,568 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2a8fea4315254ed4ac54cbb5a9aa3953 2023-07-22T12:34:22,572 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-07-22T12:34:22,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:34:22,574 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1909): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:22,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1622): Region close journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:22,575 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(163): Closed 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:22,576 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSED 2023-07-22T12:34:22,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-07-22T12:34:22,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 592 msec 2023-07-22T12:34:22,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=47, resume processing ppid=46 2023-07-22T12:34:22,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, ppid=46, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN in 597 msec 2023-07-22T12:34:22,580 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029262580"}]},"ts":"1690029262580"} 2023-07-22T12:34:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:34:22,581 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:34:22,583 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:34:22,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, state=SUCCESS; DisableTableProcedure table=backup:system in 617 msec 2023-07-22T12:34:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-07-22T12:34:23,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029263092 (current time:1690029263092). 2023-07-22T12:34:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:23,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690029263091 VERSION not specified, setting to 2 2023-07-22T12:34:23,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3342e8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:23,101 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51664, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:23,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-07-22T12:34:23,105 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:34:23,106 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:23,107 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:34:23,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:34:23,514 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:34:23,516 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:34:23,516 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:34:23,516 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:23,517 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0] hfiles 2023-07-22T12:34:23,517 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:23,518 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:23,519 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/2a8fea4315254ed4ac54cbb5a9aa3953, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5] hfiles 2023-07-22T12:34:23,519 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:23,519 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/2a8fea4315254ed4ac54cbb5a9aa3953 2023-07-22T12:34:23,519 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:34:23,926 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:34:23,927 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:23,927 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:23,927 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690029263091 2023-07-22T12:34:23,928 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029263091 2023-07-22T12:34:24,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:34:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:34:24,341 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:24,349 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:24,349 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029263091 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690029263091 2023-07-22T12:34:25,169 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:25,169 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-07-22T12:34:25,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029263091 table=backup:system type=FLUSH ttl=0 } in 2.0660 sec 2023-07-22T12:34:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:34:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029257825 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:34:25,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:34:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:34:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:34:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:34:25,654 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-07-22T12:34:25,654 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-07-22T12:34:25,655 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:25,661 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029257825 type: FLUSH version: 2 ttl: 0 2023-07-22T12:34:25,661 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system 2023-07-22T12:34:25,665 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:34:25,665 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:25,674 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:25,683 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/2a8fea4315254ed4ac54cbb5a9aa3953 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/2a8fea4315254ed4ac54cbb5a9aa3953 2023-07-22T12:34:25,684 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:25,684 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029257825 type: FLUSH version: 2 ttl: 0 2023-07-22T12:34:25,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029265685"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:25,688 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:34:25,688 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:25,688 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029265686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029265686"}]},"ts":"1690029265686"} 2023-07-22T12:34:25,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:34:25,690 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:34:25,690 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:25,690 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:34:25,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029257825 type: FLUSH version: 2 ttl: 0 ) in 464 msec 2023-07-22T12:34:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-07-22T12:34:25,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690029263091" 2023-07-22T12:34:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690029263091 2023-07-22T12:34:25,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:34:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=51, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:34:25,868 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:34:25,868 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029265868"}]},"ts":"1690029265868"} 2023-07-22T12:34:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:34:25,869 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:34:25,872 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:34:25,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN}] 2023-07-22T12:34:25,873 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN 2023-07-22T12:34:25,874 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:34:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:34:26,026 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:26,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:34:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:34:26,182 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:26,183 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:34:26,183 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:34:26,183 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,184 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:26,184 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,184 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,185 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,186 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName meta 2023-07-22T12:34:26,186 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:26,193 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:26,194 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:26,194 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,195 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName session 2023-07-22T12:34:26,195 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:26,202 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:26,209 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:26,209 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:26,210 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,211 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,213 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:34:26,215 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:26,216 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 8e2fda6a14d55f9b3ace42576496dba7; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11047069280, jitterRate=0.028838500380516052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:34:26,217 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:26,218 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., pid=53, masterSystemTime=1690029266179 2023-07-22T12:34:26,219 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:26,220 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:26,220 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:26,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-07-22T12:34:26,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 194 msec 2023-07-22T12:34:26,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=52, resume processing ppid=51 2023-07-22T12:34:26,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN in 351 msec 2023-07-22T12:34:26,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029266225"}]},"ts":"1690029266225"} 2023-07-22T12:34:26,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:34:26,228 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:34:26,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, state=SUCCESS; EnableTableProcedure table=backup:system in 370 msec 2023-07-22T12:34:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-07-22T12:34:26,472 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:34:26,472 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:26,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:26,481 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:26,481 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-07-22T12:34:26,483 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:26,489 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(246): Backup backup_1690029257800 failed. Backup session finished. Status: FAILURE 2023-07-22T12:34:26,489 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:26,489 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] ... 41 more 2023-07-22T12:34:26,495 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 2 2023-07-22T12:34:26,515 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406f0f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:26,516 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:26,518 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:26,522 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:26,523 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51672, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:26,576 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:26,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029266580 (current time:1690029266580). 2023-07-22T12:34:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4913a57d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:26,589 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51682, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-07-22T12:34:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:26,595 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:26,596 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:26,598 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:27,007 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:27,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7}] 2023-07-22T12:34:27,009 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:27,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-07-22T12:34:27,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:27,162 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-07-22T12:34:27,176 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/895d8ffa627f4db395facf80eecedcd1 is 256, key is session:backup_1690029257800/session:context/1690029266482/Put/seqid=0 2023-07-22T12:34:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:27,581 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:27,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:27,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/895d8ffa627f4db395facf80eecedcd1 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:27,595 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1, entries=2, sequenceid=33, filesize=5.2 K 2023-07-22T12:34:27,596 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 434ms, sequenceid=33, compaction requested=true 2023-07-22T12:34:27,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:27,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. for snapshot_backup_system completed. 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0] hfiles 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 for snapshot=snapshot_backup_system 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1] hfiles 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b for snapshot=snapshot_backup_system 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 for snapshot=snapshot_backup_system 2023-07-22T12:34:27,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1 for snapshot=snapshot_backup_system 2023-07-22T12:34:27,607 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:27,608 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-07-22T12:34:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=55 2023-07-22T12:34:27,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:27,609 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:27,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-07-22T12:34:27,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 in 603 msec 2023-07-22T12:34:27,612 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:27,613 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:27,614 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:27,614 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:27,615 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:28,031 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:28,039 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:28,040 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:28,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:28,858 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:28,858 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-07-22T12:34:28,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2670 sec 2023-07-22T12:34:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:34:30,700 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(115): Backup backup_1690029266553 started at 1690029270699. 2023-07-22T12:34:30,701 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(119): Backup session backup_1690029266553 has been started. 2023-07-22T12:34:30,703 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-16900292258701, test-1690029225870] 2023-07-22T12:34:30,705 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(76): StartCode 1690029221601for backupID backup_1690029266553 2023-07-22T12:34:30,706 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:34:30,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:30,707 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:30,707 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:30,708 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:30,708 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:30,708 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:30,710 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,710 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:30,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:30,711 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:30,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:30,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:30,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:30,711 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:30,712 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:30,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:30,712 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:30,712 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:30,713 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:30,713 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:30,713 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:30,715 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:30,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:30,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:30,715 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:30,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:30,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:30,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:30,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:30,716 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:30,716 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@39035329[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:30,716 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:30,716 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:30,717 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:30,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:30,717 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:30,718 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:30,718 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:30,718 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,718 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029241532 highest: 1690029241532 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,718 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1:(num 1690029241509) roll requested 2023-07-22T12:34:30,729 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718, exclude list is [], retry=0 2023-07-22T12:34:30,733 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:30,735 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 2023-07-22T12:34:30,735 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:30,735 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 is not closed yet, will try archiving it next time 2023-07-22T12:34:30,735 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0:(num 1690029241532) roll requested 2023-07-22T12:34:30,737 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 not finished, retry = 0 2023-07-22T12:34:30,746 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735, exclude list is [], retry=0 2023-07-22T12:34:30,749 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:30,751 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 with entries=124, filesize=18.41 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 2023-07-22T12:34:30,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:30,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 is not closed yet, will try archiving it next time 2023-07-22T12:34:30,752 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta:.meta(num 1690029241551) roll requested 2023-07-22T12:34:30,752 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 2023-07-22T12:34:30,764 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta, exclude list is [], retry=0 2023-07-22T12:34:30,767 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:30,769 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta 2023-07-22T12:34:30,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:30,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:30,771 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta not finished, retry = 0 2023-07-22T12:34:30,778 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-07-22T12:34:30,778 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029270735 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,786 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:30,786 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,42129,1690029216993' in zk 2023-07-22T12:34:30,789 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:30,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,789 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:30,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:30,789 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:30,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:30,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:30,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:30,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:30,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:30,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:30,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,792 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,42129,1690029216993': 2023-07-22T12:34:30,793 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:30,793 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:30,793 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:30,793 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:30,793 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:30,793 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:30,796 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:30,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:30,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:30,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:30,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:30,796 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:30,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:30,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:30,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:30,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:30,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:30,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:30,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:30,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:30,799 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:30,805 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:30,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:30,805 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:30,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:30,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:30,806 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:30,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:30,909 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:42129=1690029221601} newestTimestamps: {jenkins-hbase3.apache.org:42129=1690029241532} 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:30,912 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 2023-07-22T12:34:30,913 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 2023-07-22T12:34:31,323 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029225870 target: hdfs://localhost:33927/backupUT/backup_1690029266553/default/test-1690029225870 descriptor: 'test-1690029225870', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:31,323 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:34:31,325 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(151): Starting to write region info for table test-1690029225870 2023-07-22T12:34:31,730 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(158): Finished writing region info for table test-1690029225870 2023-07-22T12:34:31,800 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:34:31,800 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:34:32,141 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900292258701 target: hdfs://localhost:33927/backupUT/backup_1690029266553/ns2/test-16900292258701 descriptor: 'ns2:test-16900292258701', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:32,141 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:34:32,143 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900292258701 2023-07-22T12:34:32,548 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900292258701 2023-07-22T12:34:32,557 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33927/backupUT/.tmp/backup_1690029266553 from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532;hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:32,566 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d1ddb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:32,568 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:32,569 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52206, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:32,585 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:34:32,585 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900292258701 2023-07-22T12:34:32,590 DEBUG [hconnection-0x6af4dbe6-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:32,592 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43300, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:32,594 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900292258701: ns2:test-16900292258701; 2023-07-22T12:34:32,595 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029225870 2023-07-22T12:34:32,598 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029225870: test-1690029225870; 2023-07-22T12:34:32,598 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:34:32,598 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/hbase-staging/partitions_c0cce979-f834-49a6-96c0-dee2d0a2e9ea 2023-07-22T12:34:33,038 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:33,038 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:33,039 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:33,039 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,000 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-5678153744453494815.jar 2023-07-22T12:34:34,001 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,001 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,002 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,002 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,003 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,003 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,004 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,004 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,004 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,005 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:34,006 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:34,006 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:34,007 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:34,007 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:34,008 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:34,008 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:34,009 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:34,010 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:34,011 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:34,011 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:34,012 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:34,014 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,015 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,015 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,016 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,016 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,016 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:34,017 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:34,017 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16900292258701,default:test-1690029225870 2023-07-22T12:34:34,018 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:34,018 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:34,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:34:34,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:34:34,974 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-6955590552575191833.jar 2023-07-22T12:34:34,996 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 for WAL files 2023-07-22T12:34:34,997 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:34:34,997 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 for WAL files 2023-07-22T12:34:34,998 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:35,079 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029275046/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:35,079 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029275046/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:35,160 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 (-9223372036854775808:9223372036854775807) length:18860 2023-07-22T12:34:35,177 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:34:35,178 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:34:35,223 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 (-9223372036854775808:9223372036854775807) length:1198 2023-07-22T12:34:35,227 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:34:35,227 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:34:35,369 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7604c65a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:35,374 DEBUG [hconnection-0x36338406-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:35,377 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43310, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:35,379 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:35,381 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-07-22T12:34:35,381 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:34:35,388 DEBUG [pool-223-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/backupUT/.tmp/backup_1690029266553/_temporary/0/_temporary/attempt_local923784095_0003_r_000000_0/ns2/test-16900292258701/f/53855209921341b59f2a4ab1287d0508 is 38, key is row-t20/f:q1/1690029257604/Put/seqid=0 2023-07-22T12:34:35,390 WARN [Thread-961 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:42129] Specified favored nodes: [jenkins-hbase3.apache.org:42129] 2023-07-22T12:34:35,429 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac48ce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:35,485 DEBUG [hconnection-0x1fb0d8dc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:35,486 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43312, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:35,488 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:35,489 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1690029225870-1-0] 2023-07-22T12:34:35,489 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:34:35,500 DEBUG [pool-223-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/backupUT/.tmp/backup_1690029266553/_temporary/0/_temporary/attempt_local923784095_0003_r_000001_0/default/test-1690029225870/f/bee45619a97d4464a4a1ccd7b0a5efbe is 59, key is row-test-1690029225870-1-10/f:q1/1690029257494/Put/seqid=0 2023-07-22T12:34:35,502 WARN [Thread-971 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:42129] Specified favored nodes: [jenkins-hbase3.apache.org:42129] 2023-07-22T12:34:36,116 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33927/backupUT 2023-07-22T12:34:36,117 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029266553 2023-07-22T12:34:36,117 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:34:36,131 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33927/backupUT/.tmp/backup_1690029266553, hdfs://localhost:33927/backupUT] 2023-07-22T12:34:37,263 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:34:37,265 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029266553 2023-07-22T12:34:37,265 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-07-22T12:34:37,265 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1704450425_0004 completed: true true 2023-07-22T12:34:37,271 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3771426 FILE: Number of bytes written=5062008 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=194825 HDFS: Number of bytes written=2368946 HDFS: Number of read operations=732 HDFS: Number of large read operations=0 HDFS: Number of write operations=275 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1368391680 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-07-22T12:34:37,271 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33927/backupUT/.tmp/backup_1690029266553 to hdfs://localhost:33927/backupUT finished. 2023-07-22T12:34:37,273 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:34:37,274 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(240): BackupId=backup_1690029266553,startts=1690029270699,failedts=1690029277273,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_2 in testing 2023-07-22T12:34:37,274 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:34:37,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:34:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:34:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:34:37,282 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029277282"}]},"ts":"1690029277282"} 2023-07-22T12:34:37,284 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:34:37,285 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:34:37,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN}] 2023-07-22T12:34:37,287 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN 2023-07-22T12:34:37,288 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:37,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:34:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:34:37,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(119): Close 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:37,441 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1668): Closing 8e2fda6a14d55f9b3ace42576496dba7, disabling compactions & flushes 2023-07-22T12:34:37,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1690): Closing region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:37,441 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:37,441 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. after waiting 0 ms 2023-07-22T12:34:37,441 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:37,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=540 B heapSize=1.24 KB 2023-07-22T12:34:37,455 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/8f5f77b6871b48a487645893e36e331c is 116, key is rslogts:hdfs://localhost:33927/backupUT\x00jenkins-hbase3.apache.org:42129/meta:rs-log-ts/1690029270786/Put/seqid=0 2023-07-22T12:34:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:34:37,859 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=112 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/8f5f77b6871b48a487645893e36e331c 2023-07-22T12:34:37,865 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f5f77b6871b48a487645893e36e331c 2023-07-22T12:34:37,867 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/276837f6833e4d94849c00e97dac9130 is 218, key is session:backup_1690029266553/session:context/1690029277264/Put/seqid=0 2023-07-22T12:34:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:34:38,272 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/276837f6833e4d94849c00e97dac9130 2023-07-22T12:34:38,278 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 276837f6833e4d94849c00e97dac9130 2023-07-22T12:34:38,279 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/8f5f77b6871b48a487645893e36e331c as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8f5f77b6871b48a487645893e36e331c 2023-07-22T12:34:38,284 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8f5f77b6871b48a487645893e36e331c, entries=1, sequenceid=39, filesize=5.3 K 2023-07-22T12:34:38,285 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/276837f6833e4d94849c00e97dac9130 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/276837f6833e4d94849c00e97dac9130 2023-07-22T12:34:38,292 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/276837f6833e4d94849c00e97dac9130, entries=1, sequenceid=39, filesize=5.2 K 2023-07-22T12:34:38,293 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(3022): Finished flush of dataSize ~540 B/540, heapSize ~1.21 KB/1240, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 852ms, sequenceid=39, compaction requested=true 2023-07-22T12:34:38,295 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:38,297 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8f5f77b6871b48a487645893e36e331c 2023-07-22T12:34:38,300 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:38,302 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:38,304 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:38,306 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 276837f6833e4d94849c00e97dac9130 2023-07-22T12:34:38,311 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=27 2023-07-22T12:34:38,311 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:34:38,313 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1909): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:38,313 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1622): Region close journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:38,314 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(163): Closed 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:38,315 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSED 2023-07-22T12:34:38,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=58, resume processing ppid=57 2023-07-22T12:34:38,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, ppid=57, state=SUCCESS; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 1.0270 sec 2023-07-22T12:34:38,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=57, resume processing ppid=56 2023-07-22T12:34:38,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, ppid=56, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN in 1.0310 sec 2023-07-22T12:34:38,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029278319"}]},"ts":"1690029278319"} 2023-07-22T12:34:38,320 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:34:38,322 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:34:38,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0430 sec 2023-07-22T12:34:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-07-22T12:34:38,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029278395 (current time:1690029278395). 2023-07-22T12:34:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690029278394 VERSION not specified, setting to 2 2023-07-22T12:34:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25287ec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:38,404 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43318, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=59, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-07-22T12:34:38,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:34:38,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:38,411 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:34:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:34:38,819 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:34:38,820 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:34:38,820 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:34:38,820 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:38,822 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8f5f77b6871b48a487645893e36e331c] hfiles 2023-07-22T12:34:38,822 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:38,822 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8f5f77b6871b48a487645893e36e331c 2023-07-22T12:34:38,822 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:38,824 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/276837f6833e4d94849c00e97dac9130, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5] hfiles 2023-07-22T12:34:38,824 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:38,824 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/276837f6833e4d94849c00e97dac9130 2023-07-22T12:34:38,824 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:38,824 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:34:39,231 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:34:39,232 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:39,233 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:39,233 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690029278394 2023-07-22T12:34:39,233 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029278394 2023-07-22T12:34:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:34:39,645 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:39,653 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:39,654 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029278394 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690029278394 2023-07-22T12:34:40,347 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 855ade4dc43fb86bf7ef486829b4203f changed from -1.0 to 0.0, refreshing cache 2023-07-22T12:34:40,472 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:40,472 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-07-22T12:34:40,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029278394 table=backup:system type=FLUSH ttl=0 } in 2.0670 sec 2023-07-22T12:34:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:34:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=60, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029266580 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:34:40,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:34:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:34:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:34:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:34:40,937 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-07-22T12:34:40,937 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-07-22T12:34:40,938 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:40,940 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029266580 type: FLUSH version: 2 ttl: 0 2023-07-22T12:34:40,941 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system 2023-07-22T12:34:40,943 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:34:40,943 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:40,951 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8f5f77b6871b48a487645893e36e331c to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8f5f77b6871b48a487645893e36e331c 2023-07-22T12:34:40,951 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:40,955 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/276837f6833e4d94849c00e97dac9130 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/276837f6833e4d94849c00e97dac9130 2023-07-22T12:34:40,955 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:40,955 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029266580 type: FLUSH version: 2 ttl: 0 2023-07-22T12:34:40,956 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029280956"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:40,958 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:34:40,958 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:40,959 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029280957"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029280957"}]},"ts":"1690029280957"} 2023-07-22T12:34:40,960 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:34:40,960 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:34:40,960 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:40,960 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:34:40,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029266580 type: FLUSH version: 2 ttl: 0 ) in 439 msec 2023-07-22T12:34:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-07-22T12:34:41,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690029278394" 2023-07-22T12:34:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690029278394 2023-07-22T12:34:41,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:34:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=61, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:34:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-07-22T12:34:41,138 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:34:41,138 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029281138"}]},"ts":"1690029281138"} 2023-07-22T12:34:41,139 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:34:41,141 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:34:41,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN}] 2023-07-22T12:34:41,143 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN 2023-07-22T12:34:41,143 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:34:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-07-22T12:34:41,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:41,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:34:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-07-22T12:34:41,451 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(130): Open backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:41,451 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:34:41,452 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:34:41,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(888): Instantiated backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:41,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7296): checking encryption for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7299): checking classloading for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,454 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,454 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName meta 2023-07-22T12:34:41,455 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:41,461 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:41,461 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:41,461 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,462 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName session 2023-07-22T12:34:41,462 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:41,468 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:41,472 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:41,476 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:41,476 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:41,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,478 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:34:41,480 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1076): writing seq id for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:41,481 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): Opened 8e2fda6a14d55f9b3ace42576496dba7; next sequenceid=43; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9423889280, jitterRate=-0.12233191728591919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:34:41,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(992): Region open journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:41,483 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., pid=63, masterSystemTime=1690029281448 2023-07-22T12:34:41,486 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(403): Add compact mark for store 8e2fda6a14d55f9b3ace42576496dba7:session, priority=-2147483648, current under compaction store size is 1 2023-07-22T12:34:41,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-07-22T12:34:41,487 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-07-22T12:34:41,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:41,488 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(158): Opened backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:41,489 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPEN, openSeqNum=43, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:41,490 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16169 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-07-22T12:34:41,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-07-22T12:34:41,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 194 msec 2023-07-22T12:34:41,492 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HStore(1534): 8e2fda6a14d55f9b3ace42576496dba7/session is initiating minor compaction (all files) 2023-07-22T12:34:41,493 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 8e2fda6a14d55f9b3ace42576496dba7/session in backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:41,493 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1] into tmpdir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp, totalSize=15.8 K 2023-07-22T12:34:41,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=62, resume processing ppid=61 2023-07-22T12:34:41,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, ppid=61, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN in 351 msec 2023-07-22T12:34:41,494 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029281493"}]},"ts":"1690029281493"} 2023-07-22T12:34:41,494 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.Compactor(223): Compacting 22c35e6d7bd14d1cad28782199ed075b, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1690029237282 2023-07-22T12:34:41,494 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9192f5afe4e84de9ade12479dc9882a5, keycount=3, bloomtype=ROW, size=5.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1690029257445 2023-07-22T12:34:41,495 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:34:41,495 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.Compactor(223): Compacting 895d8ffa627f4db395facf80eecedcd1, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1690029266482 2023-07-22T12:34:41,496 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:34:41,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, state=SUCCESS; EnableTableProcedure table=backup:system in 363 msec 2023-07-22T12:34:41,520 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e2fda6a14d55f9b3ace42576496dba7#session#compaction#10 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-07-22T12:34:41,521 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/48058ab71b7c414fb0c37b22d89ac0d6 is 459, key is session:backup_1690029234345/session:context/1690029257445/Put/seqid=0 2023-07-22T12:34:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-07-22T12:34:41,740 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:34:41,741 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:41,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:41,750 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:41,750 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-07-22T12:34:41,751 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:33927/backupUT/backup_1690029266553/default/test-1690029225870 done. 2023-07-22T12:34:41,752 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(208): hdfs://localhost:33927/backupUT/backup_1690029266553/default is empty, remove it. 2023-07-22T12:34:41,752 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:33927/backupUT/backup_1690029266553/ns2/test-16900292258701 done. 2023-07-22T12:34:41,753 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(208): hdfs://localhost:33927/backupUT/backup_1690029266553/ns2 is empty, remove it. 2023-07-22T12:34:41,755 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:41,761 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(246): Backup backup_1690029266553 failed. Backup session finished. Status: FAILURE 2023-07-22T12:34:41,761 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:41,761 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] ... 41 more 2023-07-22T12:34:41,766 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 3 2023-07-22T12:34:41,787 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c31dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:41,788 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:41,790 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:41,793 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:41,795 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50348, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:41,844 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:41,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029281848 (current time:1690029281848). 2023-07-22T12:34:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bd4664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:41,858 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50356, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:34:41,861 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:41,862 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:41,864 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:41,936 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:34:41,937 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/48058ab71b7c414fb0c37b22d89ac0d6 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:34:41,951 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:34:41,952 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 8e2fda6a14d55f9b3ace42576496dba7/session of 8e2fda6a14d55f9b3ace42576496dba7 into 48058ab71b7c414fb0c37b22d89ac0d6(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-07-22T12:34:41,952 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:41,952 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., storeName=8e2fda6a14d55f9b3ace42576496dba7/session, priority=13, startTime=1690029281485; duration=0sec 2023-07-22T12:34:41,953 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-07-22T12:34:41,953 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e2fda6a14d55f9b3ace42576496dba7:session 2023-07-22T12:34:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:42,270 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:42,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7}] 2023-07-22T12:34:42,273 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:42,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-07-22T12:34:42,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:42,424 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-07-22T12:34:42,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/3556a936b3ea4ad2aacaff71750e3543 is 256, key is session:backup_1690029266553/session:context/1690029281754/Put/seqid=0 2023-07-22T12:34:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:42,834 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:34:42,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:34:42,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/3556a936b3ea4ad2aacaff71750e3543 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:34:42,845 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543, entries=2, sequenceid=49, filesize=5.2 K 2023-07-22T12:34:42,846 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 422ms, sequenceid=49, compaction requested=false 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. for snapshot_backup_system completed. 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0] hfiles 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 for snapshot=snapshot_backup_system 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543] hfiles 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 for snapshot=snapshot_backup_system 2023-07-22T12:34:42,847 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 for snapshot=snapshot_backup_system 2023-07-22T12:34:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:43,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:43,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-07-22T12:34:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=65 2023-07-22T12:34:43,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:43,257 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:43,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=65, resume processing ppid=64 2023-07-22T12:34:43,260 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:43,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 in 987 msec 2023-07-22T12:34:43,260 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:43,261 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:43,261 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:43,263 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:43,677 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:43,685 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:43,685 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:43,867 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:44,504 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:44,504 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:34:44,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6450 sec 2023-07-22T12:34:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:34:45,968 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(115): Backup backup_1690029281818 started at 1690029285968. 2023-07-22T12:34:45,970 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(119): Backup session backup_1690029281818 has been started. 2023-07-22T12:34:45,971 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-16900292258701, test-1690029225870] 2023-07-22T12:34:45,974 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(76): StartCode 1690029221601for backupID backup_1690029281818 2023-07-22T12:34:45,974 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:34:45,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:34:45,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:34:45,975 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:34:45,975 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:34:45,976 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:34:45,976 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:34:45,977 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:45,978 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:45,978 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:45,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:45,978 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:34:45,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:45,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:34:45,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:34:45,979 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:34:45,979 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:34:45,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:34:45,979 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:34:45,980 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:34:45,980 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:34:45,980 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:34:45,980 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:34:45,982 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:45,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:45,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:45,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:45,983 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:34:45,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:45,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:45,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:45,984 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:45,984 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:45,984 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:34:45,984 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7ba9de9c[Count = 0] remaining members to acquire global barrier 2023-07-22T12:34:45,984 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:34:45,984 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:45,985 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:45,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:34:45,986 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:34:45,986 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:34:45,986 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:45,986 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029270735 highest: 1690029270735 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:45,986 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1:(num 1690029270718) roll requested 2023-07-22T12:34:45,986 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:34:45,998 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986, exclude list is [], retry=0 2023-07-22T12:34:46,002 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:46,003 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 2023-07-22T12:34:46,003 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:46,003 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 is not closed yet, will try archiving it next time 2023-07-22T12:34:46,003 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0:(num 1690029270735) roll requested 2023-07-22T12:34:46,007 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 2023-07-22T12:34:46,016 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003, exclude list is [], retry=0 2023-07-22T12:34:46,020 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:46,022 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 with entries=13, filesize=3.56 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 2023-07-22T12:34:46,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:46,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 is not closed yet, will try archiving it next time 2023-07-22T12:34:46,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta:.meta(num 1690029270752) roll requested 2023-07-22T12:34:46,034 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029286022.meta, exclude list is [], retry=0 2023-07-22T12:34:46,038 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:34:46,039 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029286022.meta 2023-07-22T12:34:46,039 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:34:46,039 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta is not closed yet, will try archiving it next time 2023-07-22T12:34:46,046 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-07-22T12:34:46,046 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029286003 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:46,054 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:34:46,054 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,42129,1690029216993' in zk 2023-07-22T12:34:46,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:46,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:46,058 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:34:46,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:46,058 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:46,058 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:34:46,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:46,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:46,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:46,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:46,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:46,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:46,061 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:46,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,42129,1690029216993': 2023-07-22T12:34:46,062 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:34:46,062 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:34:46,062 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:34:46,062 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:34:46,062 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:34:46,062 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:34:46,063 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:34:46,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:46,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:46,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:34:46,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:34:46,064 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:46,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:34:46,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:46,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:34:46,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:46,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:34:46,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:46,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:46,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:34:46,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:34:46,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:46,069 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:34:46,069 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:34:46,070 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:34:46,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:34:46,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:34:46,070 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:34:46,070 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:34:46,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:34:46,173 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:42129=1690029221601} newestTimestamps: {jenkins-hbase3.apache.org:42129=1690029270735} 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029286022.meta 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:46,175 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 2023-07-22T12:34:46,176 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 2023-07-22T12:34:46,176 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 2023-07-22T12:34:46,586 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029225870 target: hdfs://localhost:33927/backupUT/backup_1690029281818/default/test-1690029225870 descriptor: 'test-1690029225870', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:46,586 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:34:46,588 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(151): Starting to write region info for table test-1690029225870 2023-07-22T12:34:46,993 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(158): Finished writing region info for table test-1690029225870 2023-07-22T12:34:47,404 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900292258701 target: hdfs://localhost:33927/backupUT/backup_1690029281818/ns2/test-16900292258701 descriptor: 'ns2:test-16900292258701', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:34:47,404 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:34:47,406 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900292258701 2023-07-22T12:34:47,813 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900292258701 2023-07-22T12:34:47,818 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33927/backupUT/.tmp/backup_1690029281818 from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532;hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735;hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:47,824 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8701ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:47,825 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:47,826 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:47,828 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:34:47,828 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900292258701 2023-07-22T12:34:47,831 DEBUG [hconnection-0x9178e47-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:47,833 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50366, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:47,834 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900292258701: ns2:test-16900292258701; 2023-07-22T12:34:47,835 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029225870 2023-07-22T12:34:47,836 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029225870: test-1690029225870; 2023-07-22T12:34:47,836 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:34:47,836 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/hbase-staging/partitions_36e6ac4c-282e-47d4-81ab-a9b7bcb264f6 2023-07-22T12:34:48,243 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:48,244 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:48,244 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:48,245 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,203 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-2583266292230010420.jar 2023-07-22T12:34:49,203 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,204 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,204 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,205 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,205 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,205 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,206 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,206 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,206 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,207 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:49,208 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:34:49,208 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:34:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:34:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:34:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:34:49,210 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:34:49,210 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:34:49,211 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:34:49,211 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:34:49,212 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:34:49,212 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:34:49,213 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,213 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,213 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:34:49,215 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:34:49,215 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16900292258701,default:test-1690029225870 2023-07-22T12:34:49,215 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:49,216 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:34:50,194 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-4912990667601302123.jar 2023-07-22T12:34:50,216 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 for WAL files 2023-07-22T12:34:50,218 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:34:50,218 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 for WAL files 2023-07-22T12:34:50,219 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 2023-07-22T12:34:50,219 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 for WAL files 2023-07-22T12:34:50,219 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:34:50,296 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029290261/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:34:50,296 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029290261/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:34:50,359 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 (-9223372036854775808:9223372036854775807) length:18860 2023-07-22T12:34:50,366 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:34:50,366 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:34:50,389 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 (-9223372036854775808:9223372036854775807) length:3657 2023-07-22T12:34:50,392 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:34:50,392 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:34:50,439 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 (-9223372036854775808:9223372036854775807) length:1198 2023-07-22T12:34:50,442 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:34:50,443 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:34:50,469 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b93f16d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:50,473 DEBUG [hconnection-0x550e9f09-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:50,474 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46184, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:50,476 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:50,477 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-07-22T12:34:50,477 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:34:50,484 DEBUG [pool-246-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/backupUT/.tmp/backup_1690029281818/_temporary/0/_temporary/attempt_local1554380409_0005_r_000000_0/ns2/test-16900292258701/f/c0519c1e52f0442c8207137ec71f3b0d is 38, key is row-t20/f:q1/1690029257604/Put/seqid=0 2023-07-22T12:34:50,485 WARN [Thread-1142 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:42129] Specified favored nodes: [jenkins-hbase3.apache.org:42129] 2023-07-22T12:34:50,906 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd9bbf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:50,908 DEBUG [hconnection-0x39d2d594-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:50,909 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46198, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:50,911 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:50,912 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1690029225870-1-0] 2023-07-22T12:34:50,912 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:34:50,922 DEBUG [pool-246-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/backupUT/.tmp/backup_1690029281818/_temporary/0/_temporary/attempt_local1554380409_0005_r_000001_0/default/test-1690029225870/f/01f14ee82b1347cb92c9f38643b86003 is 59, key is row-test-1690029225870-1-10/f:q1/1690029257494/Put/seqid=0 2023-07-22T12:34:50,923 WARN [Thread-1152 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:42129] Specified favored nodes: [jenkins-hbase3.apache.org:42129] 2023-07-22T12:34:51,347 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33927/backupUT 2023-07-22T12:34:51,347 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029281818 2023-07-22T12:34:51,347 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:34:51,352 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33927/backupUT/.tmp/backup_1690029281818, hdfs://localhost:33927/backupUT] 2023-07-22T12:34:52,455 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:34:52,456 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029281818 2023-07-22T12:34:52,456 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16005 bytes copied." 2023-07-22T12:34:52,456 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local856972690_0006 completed: true true 2023-07-22T12:34:52,459 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4420550 FILE: Number of bytes written=7477776 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=306191 HDFS: Number of bytes written=2429277 HDFS: Number of read operations=1066 HDFS: Number of large read operations=0 HDFS: Number of write operations=365 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1378877440 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16005 Bytes Expected=16005 Files Copied=3 DIR_COPY=7 2023-07-22T12:34:52,459 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33927/backupUT/.tmp/backup_1690029281818 to hdfs://localhost:33927/backupUT finished. 2023-07-22T12:34:52,464 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:34:52,465 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(240): BackupId=backup_1690029281818,startts=1690029285968,failedts=1690029292465,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_3 in testing 2023-07-22T12:34:52,465 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:34:52,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:34:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:34:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:34:52,474 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029292474"}]},"ts":"1690029292474"} 2023-07-22T12:34:52,475 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:34:52,477 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:34:52,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN}] 2023-07-22T12:34:52,480 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN 2023-07-22T12:34:52,480 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:52,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:34:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:34:52,633 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(119): Close 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:52,633 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1668): Closing 8e2fda6a14d55f9b3ace42576496dba7, disabling compactions & flushes 2023-07-22T12:34:52,634 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1690): Closing region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:52,634 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:52,634 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. after waiting 0 ms 2023-07-22T12:34:52,634 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:52,634 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=872 B heapSize=1.71 KB 2023-07-22T12:34:52,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/7e23c1e3e1a1492c87ef48779b5b847c is 171, key is trslm:hdfs://localhost:33927/backupUT\x00ns2:test-16900292258701/meta:log-roll-map/1690029292463/Put/seqid=0 2023-07-22T12:34:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:34:53,051 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=444 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/7e23c1e3e1a1492c87ef48779b5b847c 2023-07-22T12:34:53,056 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7e23c1e3e1a1492c87ef48779b5b847c 2023-07-22T12:34:53,059 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/d4239590dd044f14a1d938e898378c0b is 218, key is session:backup_1690029281818/session:context/1690029292455/Put/seqid=0 2023-07-22T12:34:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:34:53,463 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/d4239590dd044f14a1d938e898378c0b 2023-07-22T12:34:53,469 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d4239590dd044f14a1d938e898378c0b 2023-07-22T12:34:53,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/7e23c1e3e1a1492c87ef48779b5b847c as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/7e23c1e3e1a1492c87ef48779b5b847c 2023-07-22T12:34:53,475 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/7e23c1e3e1a1492c87ef48779b5b847c, entries=3, sequenceid=56, filesize=5.6 K 2023-07-22T12:34:53,476 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/d4239590dd044f14a1d938e898378c0b as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/d4239590dd044f14a1d938e898378c0b 2023-07-22T12:34:53,481 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/d4239590dd044f14a1d938e898378c0b, entries=1, sequenceid=56, filesize=5.2 K 2023-07-22T12:34:53,482 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(3022): Finished flush of dataSize ~872 B/872, heapSize ~1.68 KB/1720, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 848ms, sequenceid=56, compaction requested=true 2023-07-22T12:34:53,483 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:53,485 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7e23c1e3e1a1492c87ef48779b5b847c 2023-07-22T12:34:53,487 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:53,488 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:53,490 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:53,491 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1] to archive 2023-07-22T12:34:53,491 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-07-22T12:34:53,492 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:53,495 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/22c35e6d7bd14d1cad28782199ed075b 2023-07-22T12:34:53,495 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:53,496 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/9192f5afe4e84de9ade12479dc9882a5 2023-07-22T12:34:53,496 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:53,497 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/895d8ffa627f4db395facf80eecedcd1 2023-07-22T12:34:53,525 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:34:53,527 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:34:53,528 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: d4239590dd044f14a1d938e898378c0b 2023-07-22T12:34:53,533 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=42 2023-07-22T12:34:53,533 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:34:53,534 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1909): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:53,534 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1622): Region close journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:53,535 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(163): Closed 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:53,536 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSED 2023-07-22T12:34:53,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=68, resume processing ppid=67 2023-07-22T12:34:53,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=68, ppid=67, state=SUCCESS; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 1.0560 sec 2023-07-22T12:34:53,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=67, resume processing ppid=66 2023-07-22T12:34:53,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN in 1.0610 sec 2023-07-22T12:34:53,540 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029293540"}]},"ts":"1690029293540"} 2023-07-22T12:34:53,541 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:34:53,544 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:34:53,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0740 sec 2023-07-22T12:34:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:34:53,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029293584 (current time:1690029293584). 2023-07-22T12:34:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690029293583 VERSION not specified, setting to 2 2023-07-22T12:34:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78cc26bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:53,592 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46208, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=69, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-07-22T12:34:53,595 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-07-22T12:34:53,596 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:53,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-07-22T12:34:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-07-22T12:34:54,003 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:34:54,004 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:34:54,004 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:34:54,004 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:54,006 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/7e23c1e3e1a1492c87ef48779b5b847c] hfiles 2023-07-22T12:34:54,006 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:54,006 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/7e23c1e3e1a1492c87ef48779b5b847c 2023-07-22T12:34:54,006 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:54,007 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/d4239590dd044f14a1d938e898378c0b] hfiles 2023-07-22T12:34:54,007 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:34:54,007 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:34:54,007 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/d4239590dd044f14a1d938e898378c0b 2023-07-22T12:34:54,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:34:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-07-22T12:34:54,413 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:34:54,414 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:54,414 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:54,414 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690029293583 2023-07-22T12:34:54,415 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029293583 2023-07-22T12:34:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-07-22T12:34:54,826 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:54,833 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:54,834 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029293583 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690029293583 2023-07-22T12:34:55,650 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:55,650 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-07-22T12:34:55,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029293583 table=backup:system type=FLUSH ttl=0 } in 2.0570 sec 2023-07-22T12:34:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-07-22T12:34:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=70, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029281848 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:34:55,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:34:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:34:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:34:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:34:56,121 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-07-22T12:34:56,121 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-07-22T12:34:56,123 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:56,125 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029281848 type: FLUSH version: 2 ttl: 0 2023-07-22T12:34:56,125 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system 2023-07-22T12:34:56,128 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:34:56,128 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,133 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/7e23c1e3e1a1492c87ef48779b5b847c to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/7e23c1e3e1a1492c87ef48779b5b847c 2023-07-22T12:34:56,133 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:56,135 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/d4239590dd044f14a1d938e898378c0b to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/d4239590dd044f14a1d938e898378c0b 2023-07-22T12:34:56,135 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:56,136 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029281848 type: FLUSH version: 2 ttl: 0 2023-07-22T12:34:56,136 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029296136"}]},"ts":"9223372036854775807"} 2023-07-22T12:34:56,138 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:34:56,138 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:56,138 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029296137"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029296137"}]},"ts":"1690029296137"} 2023-07-22T12:34:56,139 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:34:56,139 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:34:56,139 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:34:56,140 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:34:56,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=70, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029281848 type: FLUSH version: 2 ttl: 0 ) in 433 msec 2023-07-22T12:34:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-07-22T12:34:56,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690029293583" 2023-07-22T12:34:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690029293583 2023-07-22T12:34:56,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:34:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=71, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:34:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:34:56,321 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:34:56,321 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029296321"}]},"ts":"1690029296321"} 2023-07-22T12:34:56,323 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:34:56,329 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:34:56,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN}] 2023-07-22T12:34:56,331 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN 2023-07-22T12:34:56,332 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:34:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:34:56,483 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:56,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:34:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:34:56,639 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(130): Open backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:56,639 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:34:56,640 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:34:56,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(888): Instantiated backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:34:56,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7296): checking encryption for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7299): checking classloading for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,641 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,642 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName meta 2023-07-22T12:34:56,642 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:56,647 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:34:56,647 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:56,647 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,648 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName session 2023-07-22T12:34:56,648 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:34:56,653 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:34:56,657 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:34:56,657 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:34:56,658 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,659 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,660 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:34:56,661 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1076): writing seq id for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:56,662 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): Opened 8e2fda6a14d55f9b3ace42576496dba7; next sequenceid=60; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10905500000, jitterRate=0.015653833746910095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:34:56,662 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(992): Region open journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:56,663 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., pid=73, masterSystemTime=1690029296636 2023-07-22T12:34:56,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:56,665 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(158): Opened backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:56,665 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPEN, openSeqNum=60, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:34:56,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=73, resume processing ppid=72 2023-07-22T12:34:56,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=73, ppid=72, state=SUCCESS; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 182 msec 2023-07-22T12:34:56,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=72, resume processing ppid=71 2023-07-22T12:34:56,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=72, ppid=71, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN in 339 msec 2023-07-22T12:34:56,670 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029296669"}]},"ts":"1690029296669"} 2023-07-22T12:34:56,671 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:34:56,672 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:34:56,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=71, state=SUCCESS; EnableTableProcedure table=backup:system in 354 msec 2023-07-22T12:34:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:34:56,924 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:34:56,924 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:34:56,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:34:56,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:34:56,931 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:34:56,931 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-07-22T12:34:56,932 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:33927/backupUT/backup_1690029281818/default/test-1690029225870 done. 2023-07-22T12:34:56,933 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(208): hdfs://localhost:33927/backupUT/backup_1690029281818/default is empty, remove it. 2023-07-22T12:34:56,934 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:33927/backupUT/backup_1690029281818/ns2/test-16900292258701 done. 2023-07-22T12:34:56,935 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(208): hdfs://localhost:33927/backupUT/backup_1690029281818/ns2 is empty, remove it. 2023-07-22T12:34:56,936 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:34:56,943 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(246): Backup backup_1690029281818 failed. Backup session finished. Status: FAILURE 2023-07-22T12:34:56,943 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:56,943 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] ... 41 more 2023-07-22T12:34:56,949 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 4 2023-07-22T12:34:56,968 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4541e41a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:56,969 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:34:56,970 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:32916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:34:56,973 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:56,975 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46214, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:57,025 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:34:57,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029297029 (current time:1690029297029). 2023-07-22T12:34:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:34:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:34:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59e8de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:34:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:34:57,038 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46230, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:34:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:34:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:34:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=74, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:34:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-07-22T12:34:57,042 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:34:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:34:57,043 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:34:57,044 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:34:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:34:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:34:57,452 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:34:57,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7}] 2023-07-22T12:34:57,454 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:57,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42129 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=75 2023-07-22T12:34:57,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:57,607 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-07-22T12:34:57,621 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/08cde507b5094c5fa3aa0a1b5b56ef22 is 256, key is session:backup_1690029281818/session:context/1690029296935/Put/seqid=0 2023-07-22T12:34:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:34:58,025 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:34:58,031 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:34:58,032 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/08cde507b5094c5fa3aa0a1b5b56ef22 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:34:58,037 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22, entries=2, sequenceid=65, filesize=5.2 K 2023-07-22T12:34:58,038 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 431ms, sequenceid=65, compaction requested=true 2023-07-22T12:34:58,038 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2520): Flush status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:34:58,038 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. for snapshot_backup_system completed. 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0] hfiles 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 for snapshot=snapshot_backup_system 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22] hfiles 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 for snapshot=snapshot_backup_system 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 for snapshot=snapshot_backup_system 2023-07-22T12:34:58,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22 for snapshot=snapshot_backup_system 2023-07-22T12:34:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:34:58,445 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:34:58,445 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=75 2023-07-22T12:34:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster(3994): Remote procedure done, pid=75 2023-07-22T12:34:58,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:58,447 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:34:58,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=75, resume processing ppid=74 2023-07-22T12:34:58,449 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:34:58,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=75, ppid=74, state=SUCCESS; SnapshotRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7 in 995 msec 2023-07-22T12:34:58,450 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:34:58,450 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:34:58,450 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:34:58,451 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:34:58,865 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:34:58,871 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:34:58,872 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:34:59,076 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:34:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:34:59,691 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:34:59,691 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-07-22T12:34:59,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=74, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6510 sec 2023-07-22T12:35:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-07-22T12:35:01,149 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(115): Backup backup_1690029297005 started at 1690029301148. 2023-07-22T12:35:01,150 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(119): Backup session backup_1690029297005 has been started. 2023-07-22T12:35:01,151 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-16900292258701, test-1690029225870] 2023-07-22T12:35:01,153 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(76): StartCode 1690029221601for backupID backup_1690029297005 2023-07-22T12:35:01,154 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-07-22T12:35:01,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:35:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:35:01,154 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:35:01,154 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:35:01,156 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:35:01,156 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:35:01,156 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:01,157 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,157 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:35:01,157 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:35:01,158 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:35:01,158 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:01,158 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:35:01,158 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:35:01,158 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:35:01,159 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:35:01,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:35:01,159 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:35:01,159 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:35:01,160 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:35:01,160 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:35:01,160 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:35:01,162 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:01,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:35:01,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:35:01,162 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:35:01,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:35:01,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:35:01,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:01,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:35:01,163 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,42129,1690029216993' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:35:01,163 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7da8f1e2[Count = 0] remaining members to acquire global barrier 2023-07-22T12:35:01,163 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:35:01,163 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:01,166 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:01,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:35:01,166 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:35:01,166 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:35:01,166 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:35:01,166 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,166 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690029286003 highest: 1690029286003 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,166 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1:(num 1690029285986) roll requested 2023-07-22T12:35:01,178 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029301167, exclude list is [], retry=0 2023-07-22T12:35:01,183 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:35:01,185 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029301167 2023-07-22T12:35:01,185 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:35:01,185 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 is not closed yet, will try archiving it next time 2023-07-22T12:35:01,185 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0:(num 1690029286003) roll requested 2023-07-22T12:35:01,188 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 2023-07-22T12:35:01,197 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029301185, exclude list is [], retry=0 2023-07-22T12:35:01,200 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:35:01,202 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 with entries=13, filesize=3.55 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029301185 2023-07-22T12:35:01,202 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:35:01,202 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 is not closed yet, will try archiving it next time 2023-07-22T12:35:01,202 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta:.meta(num 1690029286022) roll requested 2023-07-22T12:35:01,202 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 2023-07-22T12:35:01,215 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029301202.meta, exclude list is [], retry=0 2023-07-22T12:35:01,219 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK] 2023-07-22T12:35:01,220 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029286022.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029301202.meta 2023-07-22T12:35:01,220 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34885,DS-4c2c1b26-bcf3-467c-834a-3d233a3de90b,DISK]] 2023-07-22T12:35:01,220 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029286022.meta is not closed yet, will try archiving it next time 2023-07-22T12:35:01,227 DEBUG [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-07-22T12:35:01,227 INFO [rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690029301185 on jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,234 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:35:01,234 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,42129,1690029216993' in zk 2023-07-22T12:35:01,240 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:35:01,240 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:35:01,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,240 DEBUG [member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:35:01,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:35:01,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:35:01,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:35:01,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:35:01,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:01,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:35:01,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:01,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,244 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,42129,1690029216993': 2023-07-22T12:35:01,244 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,42129,1690029216993' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:35:01,244 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:35:01,244 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:35:01,244 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:35:01,244 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:35:01,244 INFO [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:35:01,246 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:35:01,246 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:35:01,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:01,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:35:01,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:01,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:35:01,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:35:01,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:01,252 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:35:01,252 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:35:01,252 DEBUG [(jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:35:01,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:35:01,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:35:01,253 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:35:01,253 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:35:01,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:35:01,356 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:42129=1690029221601} newestTimestamps: {jenkins-hbase3.apache.org:42129=1690029286003} 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029219975.meta 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029241551.meta 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029270752.meta 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029286022.meta 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029301202.meta 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 2023-07-22T12:35:01,358 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029301185 2023-07-22T12:35:01,359 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:35:01,359 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029301167 2023-07-22T12:35:01,359 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029219003 2023-07-22T12:35:01,360 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029270735 2023-07-22T12:35:01,360 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029270718 2023-07-22T12:35:01,360 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029285986 2023-07-22T12:35:01,369 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(146): Attempting to copy table info for:test-1690029225870 target: hdfs://localhost:33927/backupUT/backup_1690029297005/default/test-1690029225870 descriptor: 'test-1690029225870', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:01,369 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:35:01,371 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(151): Starting to write region info for table test-1690029225870 2023-07-22T12:35:01,776 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(158): Finished writing region info for table test-1690029225870 2023-07-22T12:35:02,186 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16900292258701 target: hdfs://localhost:33927/backupUT/backup_1690029297005/ns2/test-16900292258701 descriptor: 'ns2:test-16900292258701', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:35:02,186 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-07-22T12:35:02,188 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16900292258701 2023-07-22T12:35:02,601 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16900292258701 2023-07-22T12:35:02,605 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33927/backupUT/.tmp/backup_1690029297005 from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532;hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003;hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:35:02,613 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f05be0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:02,613 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:35:02,615 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:46152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:35:02,617 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-07-22T12:35:02,617 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16900292258701 2023-07-22T12:35:02,620 DEBUG [hconnection-0x3915d64c-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:02,621 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41338, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:02,622 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16900292258701: ns2:test-16900292258701; 2023-07-22T12:35:02,622 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1690029225870 2023-07-22T12:35:02,626 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1690029225870: test-1690029225870; 2023-07-22T12:35:02,626 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-07-22T12:35:02,626 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/hbase-staging/partitions_ded53597-38c8-4c0c-b078-5cdedd79ec44 2023-07-22T12:35:03,032 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:03,033 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:03,034 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:03,034 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,033 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-5867882242541355874.jar 2023-07-22T12:35:04,033 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,033 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,034 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,034 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,035 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,035 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,035 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,036 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,036 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,037 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:35:04,037 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-07-22T12:35:04,038 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-07-22T12:35:04,038 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-07-22T12:35:04,039 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-07-22T12:35:04,039 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-07-22T12:35:04,039 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-07-22T12:35:04,040 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-07-22T12:35:04,040 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-07-22T12:35:04,040 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-07-22T12:35:04,041 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-07-22T12:35:04,041 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-07-22T12:35:04,042 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,042 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,043 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,043 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,043 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,043 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-07-22T12:35:04,044 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-07-22T12:35:04,044 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16900292258701,default:test-1690029225870 2023-07-22T12:35:04,044 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:04,045 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-07-22T12:35:04,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:35:04,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:35:05,006 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop-6646201620573137793.jar 2023-07-22T12:35:05,031 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 for WAL files 2023-07-22T12:35:05,033 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 2023-07-22T12:35:05,033 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 for WAL files 2023-07-22T12:35:05,034 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 2023-07-22T12:35:05,034 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 for WAL files 2023-07-22T12:35:05,035 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 2023-07-22T12:35:05,110 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029305075/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-07-22T12:35:05,110 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/mapreduce.cluster.local.dir/1690029305075/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-07-22T12:35:05,168 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029241532 (-9223372036854775808:9223372036854775807) length:18860 2023-07-22T12:35:05,173 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:35:05,173 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:35:05,220 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0.1690029286003 (-9223372036854775808:9223372036854775807) length:3644 2023-07-22T12:35:05,223 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:35:05,223 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:35:05,244 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 for hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1.1690029241509 (-9223372036854775808:9223372036854775807) length:1198 2023-07-22T12:35:05,247 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-07-22T12:35:05,247 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-07-22T12:35:05,266 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c82441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:05,270 DEBUG [hconnection-0x3a97b0ac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:05,272 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41348, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:05,274 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:05,275 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-07-22T12:35:05,275 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:35:05,279 DEBUG [pool-269-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/backupUT/.tmp/backup_1690029297005/_temporary/0/_temporary/attempt_local483900874_0007_r_000000_0/ns2/test-16900292258701/f/4a32a8fa20d148c9a654ab6c7c11317a is 38, key is row-t20/f:q1/1690029257604/Put/seqid=0 2023-07-22T12:35:05,280 WARN [Thread-1317 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:42129] Specified favored nodes: [jenkins-hbase3.apache.org:42129] 2023-07-22T12:35:05,701 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f938df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:05,703 DEBUG [hconnection-0x3bd97173-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:05,705 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41364, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:05,706 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:05,710 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1690029225870-1-0] 2023-07-22T12:35:05,710 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-07-22T12:35:05,717 DEBUG [pool-269-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/backupUT/.tmp/backup_1690029297005/_temporary/0/_temporary/attempt_local483900874_0007_r_000001_0/default/test-1690029225870/f/3268449ac1ef4bf68842b3e2f07215bb is 59, key is row-test-1690029225870-1-10/f:q1/1690029257494/Put/seqid=0 2023-07-22T12:35:05,719 WARN [Thread-1327 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:42129] Specified favored nodes: [jenkins-hbase3.apache.org:42129] 2023-07-22T12:35:06,149 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33927/backupUT 2023-07-22T12:35:06,149 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1690029297005 2023-07-22T12:35:06,149 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-07-22T12:35:06,154 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33927/backupUT/.tmp/backup_1690029297005, hdfs://localhost:33927/backupUT] 2023-07-22T12:35:06,767 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-07-22T12:35:06,769 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1690029297005 2023-07-22T12:35:06,769 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-07-22T12:35:06,770 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local271965637_0008 completed: true true 2023-07-22T12:35:06,774 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5069673 FILE: Number of bytes written=9890605 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=403096 HDFS: Number of bytes written=2483755 HDFS: Number of read operations=1363 HDFS: Number of large read operations=0 HDFS: Number of write operations=462 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1399848960 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-07-22T12:35:06,774 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33927/backupUT/.tmp/backup_1690029297005 to hdfs://localhost:33927/backupUT finished. 2023-07-22T12:35:06,787 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-07-22T12:35:06,788 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:35:06,788 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(240): BackupId=backup_1690029297005,startts=1690029301148,failedts=1690029306788,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_4 in testing 2023-07-22T12:35:06,788 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:35:06,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:35:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:35:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-07-22T12:35:06,798 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029306798"}]},"ts":"1690029306798"} 2023-07-22T12:35:06,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:35:06,801 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:35:06,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN}] 2023-07-22T12:35:06,803 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN 2023-07-22T12:35:06,803 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:06,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:35:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-07-22T12:35:06,957 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(119): Close 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:06,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1668): Closing 8e2fda6a14d55f9b3ace42576496dba7, disabling compactions & flushes 2023-07-22T12:35:06,957 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1690): Closing region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:06,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:06,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. after waiting 0 ms 2023-07-22T12:35:06,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:06,957 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=959 B heapSize=1.87 KB 2023-07-22T12:35:06,977 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/8dcb8e68eac04c8e9ecde012b592649a is 171, key is trslm:hdfs://localhost:33927/backupUT\x00ns2:test-16900292258701/meta:log-roll-map/1690029306778/Put/seqid=0 2023-07-22T12:35:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-07-22T12:35:07,384 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/8dcb8e68eac04c8e9ecde012b592649a 2023-07-22T12:35:07,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8dcb8e68eac04c8e9ecde012b592649a 2023-07-22T12:35:07,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/35029051f0ab4269865a031f91ec7156 is 218, key is session:backup_1690029297005/session:context/1690029306768/Put/seqid=0 2023-07-22T12:35:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-07-22T12:35:07,798 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/35029051f0ab4269865a031f91ec7156 2023-07-22T12:35:07,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 35029051f0ab4269865a031f91ec7156 2023-07-22T12:35:07,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/meta/8dcb8e68eac04c8e9ecde012b592649a as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8dcb8e68eac04c8e9ecde012b592649a 2023-07-22T12:35:07,810 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8dcb8e68eac04c8e9ecde012b592649a, entries=4, sequenceid=73, filesize=5.7 K 2023-07-22T12:35:07,811 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/35029051f0ab4269865a031f91ec7156 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/35029051f0ab4269865a031f91ec7156 2023-07-22T12:35:07,816 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/35029051f0ab4269865a031f91ec7156, entries=1, sequenceid=73, filesize=5.2 K 2023-07-22T12:35:07,817 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~959 B/959, heapSize ~1.84 KB/1880, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 860ms, sequenceid=73, compaction requested=true 2023-07-22T12:35:07,820 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:35:07,824 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8dcb8e68eac04c8e9ecde012b592649a 2023-07-22T12:35:07,825 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:35:07,825 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:35:07,825 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:35:07,825 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 35029051f0ab4269865a031f91ec7156 2023-07-22T12:35:07,828 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/recovered.edits/76.seqid, newMaxSeqId=76, maxSeqId=59 2023-07-22T12:35:07,829 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:07,829 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1909): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:07,829 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1622): Region close journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:35:07,831 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(163): Closed 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:07,831 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=CLOSED 2023-07-22T12:35:07,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=78, resume processing ppid=77 2023-07-22T12:35:07,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=78, ppid=77, state=SUCCESS; CloseRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 1.0270 sec 2023-07-22T12:35:07,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=77, resume processing ppid=76 2023-07-22T12:35:07,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=77, ppid=76, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, UNASSIGN in 1.0330 sec 2023-07-22T12:35:07,835 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029307835"}]},"ts":"1690029307835"} 2023-07-22T12:35:07,836 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:35:07,838 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:35:07,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=76, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0440 sec 2023-07-22T12:35:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-07-22T12:35:07,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690029307912 (current time:1690029307912). 2023-07-22T12:35:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:35:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690029307911 VERSION not specified, setting to 2 2023-07-22T12:35:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fac6ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:35:07,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:35:07,923 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41374, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:35:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:07,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:35:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-07-22T12:35:07,927 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:35:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:35:07,928 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:35:07,930 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:35:08,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:35:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:35:08,344 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:35:08,345 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:35:08,345 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:35:08,345 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:08,347 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8dcb8e68eac04c8e9ecde012b592649a] hfiles 2023-07-22T12:35:08,347 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:35:08,347 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8dcb8e68eac04c8e9ecde012b592649a 2023-07-22T12:35:08,347 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:08,349 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/35029051f0ab4269865a031f91ec7156, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6] hfiles 2023-07-22T12:35:08,349 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:35:08,349 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/35029051f0ab4269865a031f91ec7156 2023-07-22T12:35:08,349 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:35:08,349 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:35:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:35:08,755 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:35:08,756 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:35:08,757 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:35:08,757 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690029307911 2023-07-22T12:35:08,758 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029307911 2023-07-22T12:35:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:35:09,188 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:35:09,196 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:35:09,197 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690029307911 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690029307911 2023-07-22T12:35:10,014 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:35:10,014 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-07-22T12:35:10,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690029307911 table=backup:system type=FLUSH ttl=0 } in 2.0890 sec 2023-07-22T12:35:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-07-22T12:35:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=80, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029297029 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:35:10,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:35:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=80 2023-07-22T12:35:10,056 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-07-22T12:35:10,056 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-07-22T12:35:10,057 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:35:10,059 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029297029 type: FLUSH version: 2 ttl: 0 2023-07-22T12:35:10,059 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system 2023-07-22T12:35:10,062 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:35:10,062 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,068 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8dcb8e68eac04c8e9ecde012b592649a to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/8dcb8e68eac04c8e9ecde012b592649a 2023-07-22T12:35:10,068 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:10,072 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/35029051f0ab4269865a031f91ec7156 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/35029051f0ab4269865a031f91ec7156 2023-07-22T12:35:10,072 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:10,072 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029297029 type: FLUSH version: 2 ttl: 0 2023-07-22T12:35:10,073 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690029310073"}]},"ts":"9223372036854775807"} 2023-07-22T12:35:10,075 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:35:10,075 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:35:10,075 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690029310074"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690029310074"}]},"ts":"1690029310074"} 2023-07-22T12:35:10,076 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:35:10,076 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:35:10,076 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:35:10,076 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:35:10,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=80, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690029297029 type: FLUSH version: 2 ttl: 0 ) in 34 msec 2023-07-22T12:35:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=80 2023-07-22T12:35:10,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690029307911" 2023-07-22T12:35:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690029307911 2023-07-22T12:35:10,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:35:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] procedure2.ProcedureExecutor(1033): Stored pid=81, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:35:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:35:10,156 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:35:10,156 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029310156"}]},"ts":"1690029310156"} 2023-07-22T12:35:10,158 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:35:10,161 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:35:10,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN}] 2023-07-22T12:35:10,163 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN 2023-07-22T12:35:10,164 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:35:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:35:10,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:10,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993}] 2023-07-22T12:35:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:35:10,473 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(130): Open backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,473 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e2fda6a14d55f9b3ace42576496dba7, NAME => 'backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:35:10,473 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:35:10,474 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,474 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(888): Instantiated backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:35:10,474 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,474 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,475 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,476 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName meta 2023-07-22T12:35:10,476 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:10,482 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/meta/2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:35:10,482 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:10,483 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,484 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e2fda6a14d55f9b3ace42576496dba7 columnFamilyName session 2023-07-22T12:35:10,484 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:35:10,490 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:35:10,495 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:35:10,501 DEBUG [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:35:10,501 INFO [StoreOpener-8e2fda6a14d55f9b3ace42576496dba7-1 {}] regionserver.HStore(324): Store=8e2fda6a14d55f9b3ace42576496dba7/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:35:10,502 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,505 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:35:10,506 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,507 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): Opened 8e2fda6a14d55f9b3ace42576496dba7; next sequenceid=77; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10393412160, jitterRate=-0.03203806281089783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:35:10,508 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:35:10,509 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., pid=83, masterSystemTime=1690029310469 2023-07-22T12:35:10,511 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-07-22T12:35:10,511 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(403): Add compact mark for store 8e2fda6a14d55f9b3ace42576496dba7:session, priority=-2147483648, current under compaction store size is 1 2023-07-22T12:35:10,511 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-07-22T12:35:10,513 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16834 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-07-22T12:35:10,513 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HStore(1534): 8e2fda6a14d55f9b3ace42576496dba7/session is initiating minor compaction (all files) 2023-07-22T12:35:10,513 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 8e2fda6a14d55f9b3ace42576496dba7/session in backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,513 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,513 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(158): Opened backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,513 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22] into tmpdir=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp, totalSize=16.4 K 2023-07-22T12:35:10,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=8e2fda6a14d55f9b3ace42576496dba7, regionState=OPEN, openSeqNum=77, regionLocation=jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:10,514 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.Compactor(223): Compacting 48058ab71b7c414fb0c37b22d89ac0d6, keycount=4, bloomtype=ROW, size=6.0 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1690029257445 2023-07-22T12:35:10,515 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3556a936b3ea4ad2aacaff71750e3543, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1690029281754 2023-07-22T12:35:10,516 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] compactions.Compactor(223): Compacting 08cde507b5094c5fa3aa0a1b5b56ef22, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1690029296935 2023-07-22T12:35:10,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=83, resume processing ppid=82 2023-07-22T12:35:10,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 8e2fda6a14d55f9b3ace42576496dba7, server=jenkins-hbase3.apache.org,42129,1690029216993 in 198 msec 2023-07-22T12:35:10,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=82, resume processing ppid=81 2023-07-22T12:35:10,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=8e2fda6a14d55f9b3ace42576496dba7, ASSIGN in 356 msec 2023-07-22T12:35:10,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690029310520"}]},"ts":"1690029310520"} 2023-07-22T12:35:10,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:35:10,523 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:35:10,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=81, state=SUCCESS; EnableTableProcedure table=backup:system in 372 msec 2023-07-22T12:35:10,551 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e2fda6a14d55f9b3ace42576496dba7#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-07-22T12:35:10,551 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/93ba39e9d8cf48daa2df86dc18587172 is 459, key is session:backup_1690029234345/session:context/1690029257445/Put/seqid=0 2023-07-22T12:35:10,565 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 93ba39e9d8cf48daa2df86dc18587172 2023-07-22T12:35:10,566 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/93ba39e9d8cf48daa2df86dc18587172 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/93ba39e9d8cf48daa2df86dc18587172 2023-07-22T12:35:10,572 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:35:10,572 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 8e2fda6a14d55f9b3ace42576496dba7/session of 8e2fda6a14d55f9b3ace42576496dba7 into 93ba39e9d8cf48daa2df86dc18587172(size=6.5 K), total size for store is 6.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-07-22T12:35:10,572 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:35:10,573 INFO [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., storeName=8e2fda6a14d55f9b3ace42576496dba7/session, priority=13, startTime=1690029310509; duration=0sec 2023-07-22T12:35:10,573 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-07-22T12:35:10,573 DEBUG [RS:0;jenkins-hbase3:42129-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e2fda6a14d55f9b3ace42576496dba7:session 2023-07-22T12:35:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-07-22T12:35:10,759 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:35:10,759 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:35:10,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:35:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44705 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:35:10,767 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:35:10,767 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-07-22T12:35:10,768 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:33927/backupUT/backup_1690029297005/default/test-1690029225870 done. 2023-07-22T12:35:10,769 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(208): hdfs://localhost:33927/backupUT/backup_1690029297005/default is empty, remove it. 2023-07-22T12:35:10,769 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:33927/backupUT/backup_1690029297005/ns2/test-16900292258701 done. 2023-07-22T12:35:10,770 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(208): hdfs://localhost:33927/backupUT/backup_1690029297005/ns2 is empty, remove it. 2023-07-22T12:35:10,772 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-07-22T12:35:10,777 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] impl.TableBackupClient(246): Backup backup_1690029297005 failed. Backup session finished. Status: FAILURE 2023-07-22T12:35:10,777 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:10,777 ERROR [Listener at jenkins-hbase3.apache.org/43793 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] ... 41 more 2023-07-22T12:35:10,782 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:10,801 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=601 (was 585) Potentially hanging thread: hconnection-0x49bbb496-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2083353857_16 at /127.0.0.1:60654 [Receiving block BP-958505433-172.31.12.81-1690029210898:blk_1073741959_1135] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-958505433-172.31.12.81-1690029210898:blk_1073741958_1134, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-237581074_16 at /127.0.0.1:60716 [Waiting for operation #25] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56341@0x7c416e2b-SendThread(127.0.0.1:56341) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-237581074_16 at /127.0.0.1:40138 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-958505433-172.31.12.81-1690029210898:blk_1073741960_1136, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'JobHistoryServer' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3235c85-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-237581074_16 at /127.0.0.1:40120 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-237581074_16 at /127.0.0.1:60830 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2083353857_16 at /127.0.0.1:60652 [Receiving block BP-958505433-172.31.12.81-1690029210898:blk_1073741958_1134] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:42129-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,42129,1690029216993' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,42129,1690029216993)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-237581074_16 at /127.0.0.1:60838 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x49bbb496-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,44705,1690029213996)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-237581074_16 at /127.0.0.1:40130 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2083353857_16 at /127.0.0.1:60660 [Receiving block BP-958505433-172.31.12.81-1690029210898:blk_1073741960_1136] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56341@0x7c416e2b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56341@0x7c416e2b java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$496/1747228925.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-958505433-172.31.12.81-1690029210898:blk_1073741959_1135, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=687 (was 667) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=78 (was 124), ProcessCount=169 (was 169), AvailableMemoryMB=16950 (was 16409) - AvailableMemoryMB LEAK? - 2023-07-22T12:35:10,802 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-07-22T12:35:10,810 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:35:10,810 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:10,810 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:35:10,810 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] util.JVMClusterUtil(257): Found active master hash=1898234084, stopped=false 2023-07-22T12:35:10,810 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:35:10,814 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:35:10,814 DEBUG [Listener at jenkins-hbase3.apache.org/43793 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:10,814 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,42129,1690029216993' ***** 2023-07-22T12:35:10,814 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:35:10,814 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:35:10,832 INFO [RS:0;jenkins-hbase3:42129 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5ead26ef{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:35:10,838 INFO [RS:0;jenkins-hbase3:42129 {}] server.AbstractConnector(383): Stopped ServerConnector@7ec090de{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:35:10,838 INFO [RS:0;jenkins-hbase3:42129 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:35:10,839 INFO [RS:0;jenkins-hbase3:42129 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@124c4546{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:35:10,839 INFO [RS:0;jenkins-hbase3:42129 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@15f06e61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.log.dir/,STOPPED} 2023-07-22T12:35:10,842 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:35:10,842 INFO [RS:0;jenkins-hbase3:42129 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:35:10,842 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:35:10,842 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:35:10,842 INFO [RS:0;jenkins-hbase3:42129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:35:10,842 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 21e7da9035e8749de782db2e8bf7e35d 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 9a6d02904febf9e4645bb4f3bf3e2671 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 8e2fda6a14d55f9b3ace42576496dba7 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 855ade4dc43fb86bf7ef486829b4203f 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 36c0073c74700eeabcb5149b86d5da47 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:35:10,843 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,42129,1690029216993 2023-07-22T12:35:10,843 DEBUG [RS:0;jenkins-hbase3:42129 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:10,844 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:35:10,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e50e27a27a191e84f289cbbf597a7b51, disabling compactions & flushes 2023-07-22T12:35:10,844 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:35:10,844 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:35:10,844 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:35:10,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:35:10,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. after waiting 0 ms 2023-07-22T12:35:10,844 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:35:10,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:35:10,845 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-07-22T12:35:10,845 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1583): Online Regions={e50e27a27a191e84f289cbbf597a7b51=backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51., 21e7da9035e8749de782db2e8bf7e35d=ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d., 9a6d02904febf9e4645bb4f3bf3e2671=ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671., 8e2fda6a14d55f9b3ace42576496dba7=backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7., 1588230740=hbase:meta,,1.1588230740, 855ade4dc43fb86bf7ef486829b4203f=hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f., 36c0073c74700eeabcb5149b86d5da47=test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47., 9321bfacf9c9df682580ca2a0b229466=ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.} 2023-07-22T12:35:10,845 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:35:10,845 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:35:10,845 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:35:10,845 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:35:10,845 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:35:10,848 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=18.81 KB heapSize=32.60 KB 2023-07-22T12:35:10,848 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 21e7da9035e8749de782db2e8bf7e35d, 36c0073c74700eeabcb5149b86d5da47, 855ade4dc43fb86bf7ef486829b4203f, 8e2fda6a14d55f9b3ace42576496dba7, 9321bfacf9c9df682580ca2a0b229466, 9a6d02904febf9e4645bb4f3bf3e2671, e50e27a27a191e84f289cbbf597a7b51 2023-07-22T12:35:10,851 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system_bulk/e50e27a27a191e84f289cbbf597a7b51/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:35:10,852 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:10,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:35:10,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e50e27a27a191e84f289cbbf597a7b51: 2023-07-22T12:35:10,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1690029236626.e50e27a27a191e84f289cbbf597a7b51. 2023-07-22T12:35:10,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 21e7da9035e8749de782db2e8bf7e35d, disabling compactions & flushes 2023-07-22T12:35:10,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:35:10,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:35:10,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. after waiting 0 ms 2023-07-22T12:35:10,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:35:10,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 21e7da9035e8749de782db2e8bf7e35d 1/1 column families, dataSize=170 B heapSize=816 B 2023-07-22T12:35:10,855 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:35:10,855 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:35:10,856 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/.tmp/f/b75557bcafdd42aca42dd933779196eb is 38, key is row-t20/f:q1/1690029257604/Put/seqid=0 2023-07-22T12:35:10,864 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/.tmp/f/b75557bcafdd42aca42dd933779196eb 2023-07-22T12:35:10,875 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b75557bcafdd42aca42dd933779196eb 2023-07-22T12:35:10,876 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/.tmp/info/2b92c95b74ef481ab275392de0ad7699 is 159, key is ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d./info:regioninfo/1690029229398/Put/seqid=0 2023-07-22T12:35:10,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/.tmp/f/b75557bcafdd42aca42dd933779196eb as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/b75557bcafdd42aca42dd933779196eb 2023-07-22T12:35:10,892 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.41 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/.tmp/info/2b92c95b74ef481ab275392de0ad7699 2023-07-22T12:35:10,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/f/b75557bcafdd42aca42dd933779196eb, entries=5, sequenceid=111, filesize=5.0 K 2023-07-22T12:35:10,895 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 21e7da9035e8749de782db2e8bf7e35d in 42ms, sequenceid=111, compaction requested=false 2023-07-22T12:35:10,900 DEBUG [StoreFileCloser-21e7da9035e8749de782db2e8bf7e35d-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3b01f0ac8176452fa5d133e559bec9a6 2023-07-22T12:35:10,901 DEBUG [StoreFileCloser-21e7da9035e8749de782db2e8bf7e35d-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b75557bcafdd42aca42dd933779196eb 2023-07-22T12:35:10,905 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2b92c95b74ef481ab275392de0ad7699 2023-07-22T12:35:10,906 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:35:10,914 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns2/test-16900292258701/21e7da9035e8749de782db2e8bf7e35d/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2023-07-22T12:35:10,915 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:10,916 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:35:10,916 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 21e7da9035e8749de782db2e8bf7e35d: 2023-07-22T12:35:10,916 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16900292258701,,1690029228608.21e7da9035e8749de782db2e8bf7e35d. 2023-07-22T12:35:10,917 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9a6d02904febf9e4645bb4f3bf3e2671, disabling compactions & flushes 2023-07-22T12:35:10,917 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:35:10,917 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:35:10,917 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. after waiting 0 ms 2023-07-22T12:35:10,917 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:35:10,920 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns4/test-16900292258703/9a6d02904febf9e4645bb4f3bf3e2671/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:35:10,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:10,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:35:10,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9a6d02904febf9e4645bb4f3bf3e2671: 2023-07-22T12:35:10,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16900292258703,,1690029230934.9a6d02904febf9e4645bb4f3bf3e2671. 2023-07-22T12:35:10,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8e2fda6a14d55f9b3ace42576496dba7, disabling compactions & flushes 2023-07-22T12:35:10,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. after waiting 0 ms 2023-07-22T12:35:10,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:10,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8e2fda6a14d55f9b3ace42576496dba7 2/2 column families, dataSize=296 B heapSize=960 B 2023-07-22T12:35:10,925 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/731920d19bd04e3faa511f746eb346db is 256, key is session:backup_1690029297005/session:context/1690029310771/Put/seqid=0 2023-07-22T12:35:10,931 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/.tmp/table/c34dedcac2cb4a9593006bb4e45c2c9e is 59, key is ns2:test-16900292258701/table:state/1690029229406/Put/seqid=0 2023-07-22T12:35:11,048 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 36c0073c74700eeabcb5149b86d5da47, 855ade4dc43fb86bf7ef486829b4203f, 8e2fda6a14d55f9b3ace42576496dba7, 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:35:11,249 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 36c0073c74700eeabcb5149b86d5da47, 855ade4dc43fb86bf7ef486829b4203f, 8e2fda6a14d55f9b3ace42576496dba7, 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:35:11,334 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=296 B at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/731920d19bd04e3faa511f746eb346db 2023-07-22T12:35:11,339 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/.tmp/table/c34dedcac2cb4a9593006bb4e45c2c9e 2023-07-22T12:35:11,339 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 731920d19bd04e3faa511f746eb346db 2023-07-22T12:35:11,340 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/.tmp/session/731920d19bd04e3faa511f746eb346db as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/731920d19bd04e3faa511f746eb346db 2023-07-22T12:35:11,345 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c34dedcac2cb4a9593006bb4e45c2c9e 2023-07-22T12:35:11,346 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/.tmp/info/2b92c95b74ef481ab275392de0ad7699 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/info/2b92c95b74ef481ab275392de0ad7699 2023-07-22T12:35:11,346 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/731920d19bd04e3faa511f746eb346db, entries=2, sequenceid=82, filesize=5.2 K 2023-07-22T12:35:11,347 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~296 B/296, heapSize ~688 B/688, currentSize=0 B/0 for 8e2fda6a14d55f9b3ace42576496dba7 in 425ms, sequenceid=82, compaction requested=false 2023-07-22T12:35:11,349 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2bdcfb6df3e84e6281f9cca557cdfaf0 2023-07-22T12:35:11,352 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:35:11,354 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:35:11,355 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:35:11,357 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22] to archive 2023-07-22T12:35:11,358 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-07-22T12:35:11,359 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:35:11,359 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/info/2b92c95b74ef481ab275392de0ad7699, entries=79, sequenceid=83, filesize=15.1 K 2023-07-22T12:35:11,361 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/48058ab71b7c414fb0c37b22d89ac0d6 2023-07-22T12:35:11,361 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/.tmp/table/c34dedcac2cb4a9593006bb4e45c2c9e as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/table/c34dedcac2cb4a9593006bb4e45c2c9e 2023-07-22T12:35:11,362 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:35:11,364 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/3556a936b3ea4ad2aacaff71750e3543 2023-07-22T12:35:11,364 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:35:11,366 DEBUG [StoreCloser-backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22 to hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/archive/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/session/08cde507b5094c5fa3aa0a1b5b56ef22 2023-07-22T12:35:11,368 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 93ba39e9d8cf48daa2df86dc18587172 2023-07-22T12:35:11,370 DEBUG [StoreFileCloser-8e2fda6a14d55f9b3ace42576496dba7-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 731920d19bd04e3faa511f746eb346db 2023-07-22T12:35:11,373 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/table/c34dedcac2cb4a9593006bb4e45c2c9e, entries=15, sequenceid=83, filesize=5.8 K 2023-07-22T12:35:11,374 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~18.81 KB/19263, heapSize ~32.32 KB/33096, currentSize=0 B/0 for 1588230740 in 529ms, sequenceid=83, compaction requested=false 2023-07-22T12:35:11,375 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2b92c95b74ef481ab275392de0ad7699 2023-07-22T12:35:11,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/backup/system/8e2fda6a14d55f9b3ace42576496dba7/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=76 2023-07-22T12:35:11,377 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c34dedcac2cb4a9593006bb4e45c2c9e 2023-07-22T12:35:11,379 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:11,379 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:11,379 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8e2fda6a14d55f9b3ace42576496dba7: 2023-07-22T12:35:11,379 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1690029234497.8e2fda6a14d55f9b3ace42576496dba7. 2023-07-22T12:35:11,379 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 855ade4dc43fb86bf7ef486829b4203f, disabling compactions & flushes 2023-07-22T12:35:11,379 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:35:11,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:35:11,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. after waiting 0 ms 2023-07-22T12:35:11,380 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:35:11,380 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 855ade4dc43fb86bf7ef486829b4203f 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-07-22T12:35:11,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/meta/1588230740/recovered.edits/86.seqid, newMaxSeqId=86, maxSeqId=1 2023-07-22T12:35:11,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:11,382 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:35:11,383 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:35:11,383 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:35:11,383 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:35:11,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/.tmp/info/0fda64b0f3f94b6dbd4e03e9e490c347 is 45, key is default/info:d/1690029221732/Put/seqid=0 2023-07-22T12:35:11,449 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1609): Waiting on 36c0073c74700eeabcb5149b86d5da47, 855ade4dc43fb86bf7ef486829b4203f, 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:35:11,649 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1609): Waiting on 36c0073c74700eeabcb5149b86d5da47, 855ade4dc43fb86bf7ef486829b4203f, 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:35:11,800 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/.tmp/info/0fda64b0f3f94b6dbd4e03e9e490c347 2023-07-22T12:35:11,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0fda64b0f3f94b6dbd4e03e9e490c347 2023-07-22T12:35:11,806 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/.tmp/info/0fda64b0f3f94b6dbd4e03e9e490c347 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/info/0fda64b0f3f94b6dbd4e03e9e490c347 2023-07-22T12:35:11,811 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/info/0fda64b0f3f94b6dbd4e03e9e490c347, entries=7, sequenceid=11, filesize=5.1 K 2023-07-22T12:35:11,812 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 855ade4dc43fb86bf7ef486829b4203f in 432ms, sequenceid=11, compaction requested=false 2023-07-22T12:35:11,821 DEBUG [StoreFileCloser-855ade4dc43fb86bf7ef486829b4203f-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0fda64b0f3f94b6dbd4e03e9e490c347 2023-07-22T12:35:11,849 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1579): Waiting on 3 regions to close 2023-07-22T12:35:11,849 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1583): Online Regions={855ade4dc43fb86bf7ef486829b4203f=hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f., 36c0073c74700eeabcb5149b86d5da47=test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47., 9321bfacf9c9df682580ca2a0b229466=ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466.} 2023-07-22T12:35:11,850 DEBUG [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1609): Waiting on 36c0073c74700eeabcb5149b86d5da47, 855ade4dc43fb86bf7ef486829b4203f, 9321bfacf9c9df682580ca2a0b229466 2023-07-22T12:35:11,852 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/hbase/namespace/855ade4dc43fb86bf7ef486829b4203f/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-07-22T12:35:11,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:11,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:35:11,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 855ade4dc43fb86bf7ef486829b4203f: 2023-07-22T12:35:11,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690029220360.855ade4dc43fb86bf7ef486829b4203f. 2023-07-22T12:35:11,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 36c0073c74700eeabcb5149b86d5da47, disabling compactions & flushes 2023-07-22T12:35:11,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:35:11,854 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:35:11,855 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. after waiting 0 ms 2023-07-22T12:35:11,855 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:35:11,855 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 36c0073c74700eeabcb5149b86d5da47 2/2 column families, dataSize=5.30 KB heapSize=12.88 KB 2023-07-22T12:35:11,877 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f/cbe4a78b0be4476a9249355453641a13 is 59, key is row-test-1690029225870-1-10/f:q1/1690029257494/Put/seqid=0 2023-07-22T12:35:11,883 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f/cbe4a78b0be4476a9249355453641a13 2023-07-22T12:35:11,888 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cbe4a78b0be4476a9249355453641a13 2023-07-22T12:35:11,889 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/.tmp/f/cbe4a78b0be4476a9249355453641a13 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/cbe4a78b0be4476a9249355453641a13 2023-07-22T12:35:11,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/f/cbe4a78b0be4476a9249355453641a13, entries=99, sequenceid=217, filesize=10.5 K 2023-07-22T12:35:11,895 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.30 KB/5425, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 36c0073c74700eeabcb5149b86d5da47 in 39ms, sequenceid=217, compaction requested=false 2023-07-22T12:35:11,902 DEBUG [StoreFileCloser-36c0073c74700eeabcb5149b86d5da47-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1c5b4c1fb8df4bc88bb1e0637f47f565 2023-07-22T12:35:11,903 DEBUG [StoreFileCloser-36c0073c74700eeabcb5149b86d5da47-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: cbe4a78b0be4476a9249355453641a13 2023-07-22T12:35:11,906 DEBUG [StoreFileCloser-36c0073c74700eeabcb5149b86d5da47-f3-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b7a678e43b8843f4a246072ebd55d154 2023-07-22T12:35:11,909 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/default/test-1690029225870/36c0073c74700eeabcb5149b86d5da47/recovered.edits/220.seqid, newMaxSeqId=220, maxSeqId=106 2023-07-22T12:35:11,909 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:11,910 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:35:11,910 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 36c0073c74700eeabcb5149b86d5da47: 2023-07-22T12:35:11,910 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1690029225870,,1690029226372.36c0073c74700eeabcb5149b86d5da47. 2023-07-22T12:35:11,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9321bfacf9c9df682580ca2a0b229466, disabling compactions & flushes 2023-07-22T12:35:11,911 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:35:11,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:35:11,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. after waiting 0 ms 2023-07-22T12:35:11,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:35:11,914 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/data/ns3/test-16900292258702/9321bfacf9c9df682580ca2a0b229466/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:35:11,914 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:35:11,915 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:35:11,915 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9321bfacf9c9df682580ca2a0b229466: 2023-07-22T12:35:11,915 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16900292258702,,1690029229812.9321bfacf9c9df682580ca2a0b229466. 2023-07-22T12:35:12,050 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,42129,1690029216993; all regions closed. 2023-07-22T12:35:12,054 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/WALs/jenkins-hbase3.apache.org,42129,1690029216993/jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta.1690029301202.meta not finished, retry = 0 2023-07-22T12:35:12,160 DEBUG [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(1062): Moved 5 WAL file(s) to /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs 2023-07-22T12:35:12,160 INFO [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.meta:.meta(num 1690029301202) 2023-07-22T12:35:12,166 DEBUG [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs 2023-07-22T12:35:12,166 INFO [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-0:(num 1690029301185) 2023-07-22T12:35:12,172 DEBUG [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/oldWALs 2023-07-22T12:35:12,172 INFO [RS:0;jenkins-hbase3:42129 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C42129%2C1690029216993.jenkins-hbase3.apache.org%2C42129%2C1690029216993.regiongroup-1:(num 1690029301167) 2023-07-22T12:35:12,172 DEBUG [RS:0;jenkins-hbase3:42129 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:12,172 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:35:12,173 INFO [RS:0;jenkins-hbase3:42129 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-07-22T12:35:12,173 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:35:12,174 INFO [RS:0;jenkins-hbase3:42129 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:42129 2023-07-22T12:35:12,238 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,42129,1690029216993] 2023-07-22T12:35:12,238 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,42129,1690029216993; numProcessing=1 2023-07-22T12:35:12,244 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,42129,1690029216993 expired; onlineServers=0 2023-07-22T12:35:12,244 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,44705,1690029213996' ***** 2023-07-22T12:35:12,244 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:35:12,245 DEBUG [M:0;jenkins-hbase3:44705 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b310b55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:35:12,245 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:35:12,289 INFO [M:0;jenkins-hbase3:44705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3ce1cd2d{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:35:12,289 INFO [M:0;jenkins-hbase3:44705 {}] server.AbstractConnector(383): Stopped ServerConnector@2d49367d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:35:12,289 INFO [M:0;jenkins-hbase3:44705 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:35:12,290 INFO [M:0;jenkins-hbase3:44705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@55284cf3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:35:12,291 INFO [M:0;jenkins-hbase3:44705 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@20cd50cf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/hadoop.log.dir/,STOPPED} 2023-07-22T12:35:12,291 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,44705,1690029213996 2023-07-22T12:35:12,291 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,44705,1690029213996; all regions closed. 2023-07-22T12:35:12,291 DEBUG [M:0;jenkins-hbase3:44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:12,291 INFO [M:0;jenkins-hbase3:44705 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:35:12,292 INFO [M:0;jenkins-hbase3:44705 {}] server.AbstractConnector(383): Stopped ServerConnector@1a8be828{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:35:12,292 DEBUG [M:0;jenkins-hbase3:44705 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:35:12,292 INFO [M:0;jenkins-hbase3:44705 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:35:12,293 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:35:12,293 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029218688 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690029218688,5,FailOnTimeoutGroup] 2023-07-22T12:35:12,293 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029218689 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690029218689,5,FailOnTimeoutGroup] 2023-07-22T12:35:12,293 DEBUG [M:0;jenkins-hbase3:44705 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:35:12,293 DEBUG [M:0;jenkins-hbase3:44705 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:35:12,293 INFO [M:0;jenkins-hbase3:44705 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:35:12,293 DEBUG [M:0;jenkins-hbase3:44705 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:35:12,293 INFO [M:0;jenkins-hbase3:44705 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:35:12,294 ERROR [M:0;jenkins-hbase3:44705 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-07-22T12:35:12,294 INFO [M:0;jenkins-hbase3:44705 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:35:12,294 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:35:12,295 WARN [M:0;jenkins-hbase3:44705 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:35:12,295 INFO [M:0;jenkins-hbase3:44705 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:35:12,295 INFO [M:0;jenkins-hbase3:44705 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:35:12,296 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:35:12,307 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:12,307 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:12,307 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:35:12,307 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:12,307 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=326.16 KB heapSize=390.40 KB 2023-07-22T12:35:12,325 DEBUG [M:0;jenkins-hbase3:44705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02e9b04f8a4a4a92a83b73a2ccc92d6c is 88, key is hbase:meta,,1/info:sn/1690029219688/Put/seqid=0 2023-07-22T12:35:12,329 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02e9b04f8a4a4a92a83b73a2ccc92d6c 2023-07-22T12:35:12,334 DEBUG [M:0;jenkins-hbase3:44705 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 02e9b04f8a4a4a92a83b73a2ccc92d6c 2023-07-22T12:35:12,337 INFO [RS:0;jenkins-hbase3:42129 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,42129,1690029216993; zookeeper connection closed. 2023-07-22T12:35:12,337 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3f6836ad {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3f6836ad 2023-07-22T12:35:12,338 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:35:12,347 DEBUG [M:0;jenkins-hbase3:44705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2885e7a416034e409c3988a4dceea940 is 1280, key is \x00\x00\x00\x00\x00\x00\x00\x19/proc:d/1690029234308/Put/seqid=0 2023-07-22T12:35:12,752 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=325.56 KB at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2885e7a416034e409c3988a4dceea940 2023-07-22T12:35:12,757 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2885e7a416034e409c3988a4dceea940 2023-07-22T12:35:12,757 DEBUG [M:0;jenkins-hbase3:44705 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2885e7a416034e409c3988a4dceea940 2023-07-22T12:35:12,776 DEBUG [M:0;jenkins-hbase3:44705 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b7b75575a164da0a8abdcab7f064567 is 82, key is jenkins-hbase3.apache.org,42129,1690029216993/rs:state/1690029218762/Put/seqid=0 2023-07-22T12:35:13,181 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=766 (bloomFilter=true), to=hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b7b75575a164da0a8abdcab7f064567 2023-07-22T12:35:13,186 DEBUG [M:0;jenkins-hbase3:44705 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3b7b75575a164da0a8abdcab7f064567 2023-07-22T12:35:13,187 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02e9b04f8a4a4a92a83b73a2ccc92d6c as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/02e9b04f8a4a4a92a83b73a2ccc92d6c 2023-07-22T12:35:13,192 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/02e9b04f8a4a4a92a83b73a2ccc92d6c, entries=8, sequenceid=766, filesize=5.5 K 2023-07-22T12:35:13,194 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2885e7a416034e409c3988a4dceea940 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2885e7a416034e409c3988a4dceea940 2023-07-22T12:35:13,199 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2885e7a416034e409c3988a4dceea940 2023-07-22T12:35:13,199 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2885e7a416034e409c3988a4dceea940, entries=83, sequenceid=766, filesize=26.0 K 2023-07-22T12:35:13,200 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b7b75575a164da0a8abdcab7f064567 as hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b7b75575a164da0a8abdcab7f064567 2023-07-22T12:35:13,206 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33927/user/jenkins/test-data/c3cd506c-4d50-15d3-1eae-55f23e2a6203/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b7b75575a164da0a8abdcab7f064567, entries=1, sequenceid=766, filesize=5.1 K 2023-07-22T12:35:13,207 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(3022): Finished flush of dataSize ~326.16 KB/333983, heapSize ~390.10 KB/399464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 899ms, sequenceid=766, compaction requested=false 2023-07-22T12:35:13,209 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 02e9b04f8a4a4a92a83b73a2ccc92d6c 2023-07-22T12:35:13,210 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2885e7a416034e409c3988a4dceea940 2023-07-22T12:35:13,212 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3b7b75575a164da0a8abdcab7f064567 2023-07-22T12:35:13,213 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:35:13,213 DEBUG [M:0;jenkins-hbase3:44705 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:35:13,217 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:35:13,217 INFO [M:0;jenkins-hbase3:44705 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:35:13,217 INFO [M:0;jenkins-hbase3:44705 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:35:13,217 INFO [M:0;jenkins-hbase3:44705 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:44705 2023-07-22T12:35:13,322 INFO [M:0;jenkins-hbase3:44705 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,44705,1690029213996; zookeeper connection closed. 2023-07-22T12:35:13,328 WARN [Listener at jenkins-hbase3.apache.org/43793 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:35:13,331 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:35:13,370 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:35:13,442 WARN [BP-958505433-172.31.12.81-1690029210898 heartbeating to localhost/127.0.0.1:33927 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:35:13,442 WARN [BP-958505433-172.31.12.81-1690029210898 heartbeating to localhost/127.0.0.1:33927 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-958505433-172.31.12.81-1690029210898 (Datanode Uuid 0a0352df-90fc-48e7-b46e-4ee4b5b1fc11) service to localhost/127.0.0.1:33927 2023-07-22T12:35:13,445 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/dfs/data/data1/current/BP-958505433-172.31.12.81-1690029210898 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:35:13,445 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/b3e0e4c6-7125-1ce7-2311-868f658ec82c/cluster_a6347123-2b0e-e3d5-116e-4fc1fccdc9ed/dfs/data/data2/current/BP-958505433-172.31.12.81-1690029210898 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:35:13,481 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:35:13,654 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:35:13,654 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-07-22T12:35:13,666 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:13,789 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:13,900 ERROR [Thread[Thread-277,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:35:13,900 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:14,006 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-07-22T12:35:14,011 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-07-22T12:35:14,013 ERROR [Thread[Thread-287,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:35:14,017 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:35:14,120 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:35:14,120 INFO [Listener at jenkins-hbase3.apache.org/43793 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped