2023-07-22 12:28:20,618 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-07-22 12:28:20,634 main DEBUG Took 0.013231 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-07-22 12:28:20,634 main DEBUG PluginManager 'Core' found 129 plugins 2023-07-22 12:28:20,635 main DEBUG PluginManager 'Level' found 0 plugins 2023-07-22 12:28:20,635 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-07-22 12:28:20,637 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,647 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-07-22 12:28:20,664 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,667 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,667 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,668 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,669 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,670 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,671 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,671 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,672 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,673 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,673 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,674 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-07-22 12:28:20,675 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,675 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-07-22 12:28:20,677 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-07-22 12:28:20,679 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-07-22 12:28:20,680 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-07-22 12:28:20,680 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-07-22 12:28:20,681 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-07-22 12:28:20,682 main DEBUG PluginManager 'Converter' found 47 plugins 2023-07-22 12:28:20,694 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-07-22 12:28:20,696 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-07-22 12:28:20,700 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-07-22 12:28:20,701 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-07-22 12:28:20,701 main DEBUG createAppenders(={Console}) 2023-07-22 12:28:20,702 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-07-22 12:28:20,702 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-07-22 12:28:20,702 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-07-22 12:28:20,703 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-07-22 12:28:20,703 main DEBUG OutputStream closed 2023-07-22 12:28:20,703 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-07-22 12:28:20,704 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-07-22 12:28:20,704 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-07-22 12:28:20,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-07-22 12:28:20,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-07-22 12:28:20,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-07-22 12:28:20,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-07-22 12:28:20,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-07-22 12:28:20,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-07-22 12:28:20,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-07-22 12:28:20,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-07-22 12:28:20,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-07-22 12:28:20,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-07-22 12:28:20,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-07-22 12:28:20,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-07-22 12:28:20,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-07-22 12:28:20,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-07-22 12:28:20,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-07-22 12:28:20,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-07-22 12:28:20,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-07-22 12:28:20,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-07-22 12:28:20,764 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22 12:28:20,764 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-07-22 12:28:20,765 main DEBUG Shutdown hook enabled. Registering a new one. 2023-07-22 12:28:20,766 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-07-22T12:28:20,795 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupRepair timeout: 13 mins 2023-07-22 12:28:20,800 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-07-22 12:28:20,800 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-07-22T12:28:21,147 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6 2023-07-22T12:28:21,153 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:28:21,156 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:28:21,157 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-07-22T12:28:21,162 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73, deleteOnExit=true 2023-07-22T12:28:21,162 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-07-22T12:28:21,163 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/test.cache.data in system properties and HBase conf 2023-07-22T12:28:21,163 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:28:21,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:28:21,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:28:21,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:28:21,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:28:21,280 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-07-22T12:28:21,775 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-07-22T12:28:21,782 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:28:21,783 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:28:21,783 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:28:21,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:28:21,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:28:21,785 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:28:21,785 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:28:21,786 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:28:21,786 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:28:21,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:28:21,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:28:21,787 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:28:21,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:28:21,788 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-07-22T12:28:22,301 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:28:22,308 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:28:22,624 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-07-22T12:28:22,794 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@403d1650 via org.mortbay.log.Slf4jLog 2023-07-22T12:28:22,812 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:28:22,851 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:28:22,893 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir/Jetty_localhost_36513_hdfs____xsgubi/webapp 2023-07-22T12:28:23,041 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36513 2023-07-22T12:28:23,061 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-07-22T12:28:23,061 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-07-22T12:28:23,680 WARN [Listener at localhost/42225 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:28:23,763 WARN [Listener at localhost/42225 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-07-22T12:28:23,780 WARN [Listener at localhost/42225 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:28:23,786 INFO [Listener at localhost/42225 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:28:23,792 INFO [Listener at localhost/42225 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir/Jetty_localhost_34379_datanode____m42t7/webapp 2023-07-22T12:28:23,893 INFO [Listener at localhost/42225 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34379 2023-07-22T12:28:24,229 WARN [Listener at localhost/35041 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-07-22T12:28:24,572 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-07-22T12:28:24,770 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x7bf89ab9468a5cad: Processing first storage report for DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5 from datanode 7e1ba756-ba8c-40a6-a2c9-b8dff684b55b 2023-07-22T12:28:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x7bf89ab9468a5cad: from storage DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5 node DatanodeRegistration(127.0.0.1:40609, datanodeUuid=7e1ba756-ba8c-40a6-a2c9-b8dff684b55b, infoPort=43413, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=746358468;c=1690028902390), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-07-22T12:28:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x7bf89ab9468a5cad: Processing first storage report for DS-e399914e-216e-48f4-a87f-9dd0b99f6cac from datanode 7e1ba756-ba8c-40a6-a2c9-b8dff684b55b 2023-07-22T12:28:24,772 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x7bf89ab9468a5cad: from storage DS-e399914e-216e-48f4-a87f-9dd0b99f6cac node DatanodeRegistration(127.0.0.1:40609, datanodeUuid=7e1ba756-ba8c-40a6-a2c9-b8dff684b55b, infoPort=43413, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=746358468;c=1690028902390), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-07-22T12:28:24,775 DEBUG [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6 2023-07-22T12:28:24,856 INFO [Listener at localhost/35041 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/zookeeper_0, clientPort=64849, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-07-22T12:28:24,874 INFO [Listener at localhost/35041 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=64849 2023-07-22T12:28:24,882 INFO [Listener at localhost/35041 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:24,884 INFO [Listener at localhost/35041 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:25,594 INFO [Listener at localhost/35041 {}] util.FSUtils(461): Created version file at hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 with version=8 2023-07-22T12:28:25,594 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/hbase-staging 2023-07-22T12:28:25,610 DEBUG [Listener at localhost/35041 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-07-22T12:28:25,610 DEBUG [Listener at localhost/35041 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-07-22T12:28:25,610 DEBUG [Listener at localhost/35041 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-07-22T12:28:25,610 DEBUG [Listener at localhost/35041 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-07-22T12:28:25,822 DEBUG [Listener at localhost/35041 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-07-22T12:28:25,975 INFO [Listener at localhost/35041 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-07-22T12:28:26,605 INFO [Listener at localhost/35041 {}] client.ConnectionUtils(127): master/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:28:26,651 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:28:26,652 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:28:26,652 INFO [Listener at localhost/35041 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:28:26,652 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:28:26,652 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:28:26,808 INFO [Listener at localhost/35041 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:28:26,914 DEBUG [Listener at localhost/35041 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-07-22T12:28:26,920 INFO [Listener at localhost/35041 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:28:26,968 DEBUG [Listener at localhost/35041 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 31252 (auto-detected) 2023-07-22T12:28:26,970 DEBUG [Listener at localhost/35041 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-07-22T12:28:27,010 INFO [Listener at localhost/35041 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:43215 2023-07-22T12:28:27,024 INFO [Listener at localhost/35041 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:27,026 INFO [Listener at localhost/35041 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:27,044 INFO [Listener at localhost/35041 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:43215 connecting to ZooKeeper ensemble=127.0.0.1:64849 2023-07-22T12:28:27,090 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:432150x0, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:28:27,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:43215-0x100005bf1e10000 connected 2023-07-22T12:28:27,118 DEBUG [Listener at localhost/35041 {}] zookeeper.ZKUtil(113): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:28:27,121 DEBUG [Listener at localhost/35041 {}] zookeeper.ZKUtil(113): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:28:27,125 DEBUG [Listener at localhost/35041 {}] zookeeper.ZKUtil(113): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:28:27,137 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43215 2023-07-22T12:28:27,138 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43215 2023-07-22T12:28:27,138 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43215 2023-07-22T12:28:27,139 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43215 2023-07-22T12:28:27,139 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43215 2023-07-22T12:28:27,198 INFO [Listener at localhost/35041 {}] log.Log(170): Logging initialized @7227ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-07-22T12:28:27,328 INFO [Listener at localhost/35041 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:28:27,329 INFO [Listener at localhost/35041 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:28:27,329 INFO [Listener at localhost/35041 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:28:27,331 INFO [Listener at localhost/35041 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-07-22T12:28:27,332 INFO [Listener at localhost/35041 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:28:27,332 INFO [Listener at localhost/35041 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:28:27,335 INFO [Listener at localhost/35041 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:28:27,463 INFO [Listener at localhost/35041 {}] http.HttpServer(1219): Jetty bound to port 41475 2023-07-22T12:28:27,465 INFO [Listener at localhost/35041 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:28:27,496 INFO [Listener at localhost/35041 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:28:27,499 INFO [Listener at localhost/35041 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5ac82cb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:28:27,500 INFO [Listener at localhost/35041 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:28:27,500 INFO [Listener at localhost/35041 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4f1ed2f7{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:28:27,564 INFO [Listener at localhost/35041 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:28:27,578 INFO [Listener at localhost/35041 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:28:27,578 INFO [Listener at localhost/35041 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:28:27,580 INFO [Listener at localhost/35041 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-07-22T12:28:27,588 INFO [Listener at localhost/35041 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:28:28,729 INFO [Listener at localhost/35041 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5e79011{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:28:28,740 INFO [Listener at localhost/35041 {}] server.AbstractConnector(333): Started ServerConnector@29f0e65e{HTTP/1.1, (http/1.1)}{0.0.0.0:41475} 2023-07-22T12:28:28,741 INFO [Listener at localhost/35041 {}] server.Server(415): Started @8770ms 2023-07-22T12:28:28,744 INFO [Listener at localhost/35041 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1, hbase.cluster.distributed=false 2023-07-22T12:28:28,803 INFO [Listener at localhost/35041 {}] client.ConnectionUtils(127): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-07-22T12:28:28,804 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:28:28,804 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-07-22T12:28:28,804 INFO [Listener at localhost/35041 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-07-22T12:28:28,804 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-07-22T12:28:28,804 INFO [Listener at localhost/35041 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-07-22T12:28:28,812 INFO [Listener at localhost/35041 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-07-22T12:28:28,814 INFO [Listener at localhost/35041 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-07-22T12:28:28,815 INFO [Listener at localhost/35041 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:36719 2023-07-22T12:28:28,818 INFO [Listener at localhost/35041 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-07-22T12:28:28,825 DEBUG [Listener at localhost/35041 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-07-22T12:28:28,826 INFO [Listener at localhost/35041 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:28,829 INFO [Listener at localhost/35041 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:28,831 INFO [Listener at localhost/35041 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:36719 connecting to ZooKeeper ensemble=127.0.0.1:64849 2023-07-22T12:28:28,835 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:367190x0, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-07-22T12:28:28,836 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:36719-0x100005bf1e10001 connected 2023-07-22T12:28:28,836 DEBUG [Listener at localhost/35041 {}] zookeeper.ZKUtil(113): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-07-22T12:28:28,837 DEBUG [Listener at localhost/35041 {}] zookeeper.ZKUtil(113): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-07-22T12:28:28,838 DEBUG [Listener at localhost/35041 {}] zookeeper.ZKUtil(113): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-07-22T12:28:28,842 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36719 2023-07-22T12:28:28,842 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36719 2023-07-22T12:28:28,842 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36719 2023-07-22T12:28:28,843 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36719 2023-07-22T12:28:28,843 DEBUG [Listener at localhost/35041 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36719 2023-07-22T12:28:28,846 INFO [Listener at localhost/35041 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-07-22T12:28:28,846 INFO [Listener at localhost/35041 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-07-22T12:28:28,846 INFO [Listener at localhost/35041 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-07-22T12:28:28,847 INFO [Listener at localhost/35041 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-07-22T12:28:28,847 INFO [Listener at localhost/35041 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-07-22T12:28:28,848 INFO [Listener at localhost/35041 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-07-22T12:28:28,848 INFO [Listener at localhost/35041 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-07-22T12:28:28,850 INFO [Listener at localhost/35041 {}] http.HttpServer(1219): Jetty bound to port 38831 2023-07-22T12:28:28,850 INFO [Listener at localhost/35041 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:28:28,851 INFO [Listener at localhost/35041 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:28:28,852 INFO [Listener at localhost/35041 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@24084749{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.log.dir/,AVAILABLE} 2023-07-22T12:28:28,852 INFO [Listener at localhost/35041 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:28:28,852 INFO [Listener at localhost/35041 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@433cc464{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-07-22T12:28:28,863 INFO [Listener at localhost/35041 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-07-22T12:28:28,864 INFO [Listener at localhost/35041 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-07-22T12:28:28,864 INFO [Listener at localhost/35041 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-07-22T12:28:28,865 INFO [Listener at localhost/35041 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-07-22T12:28:28,866 INFO [Listener at localhost/35041 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-07-22T12:28:28,870 INFO [Listener at localhost/35041 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@14d0d1a8{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:28:28,871 INFO [Listener at localhost/35041 {}] server.AbstractConnector(333): Started ServerConnector@63fc2aad{HTTP/1.1, (http/1.1)}{0.0.0.0:38831} 2023-07-22T12:28:28,871 INFO [Listener at localhost/35041 {}] server.Server(415): Started @8900ms 2023-07-22T12:28:28,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-07-22T12:28:28,879 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@49910570{HTTP/1.1, (http/1.1)}{0.0.0.0:44715} 2023-07-22T12:28:28,879 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8908ms 2023-07-22T12:28:28,879 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:28,890 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:28:28,890 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:28:28,892 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:28,899 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:43215 2023-07-22T12:28:28,912 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:28:28,912 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-07-22T12:28:28,913 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:28,913 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:28,914 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:28:28,916 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,43215,1690028905806 from backup master directory 2023-07-22T12:28:28,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on existing znode=/1/master 2023-07-22T12:28:28,921 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:28:28,921 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:28,921 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-07-22T12:28:28,922 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:28:28,922 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:28,925 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-07-22T12:28:28,927 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-07-22T12:28:29,011 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/hbase.id with ID: 9bfdd9fa-516f-4b07-8560-00861e2f3f2f 2023-07-22T12:28:29,051 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-07-22T12:28:29,067 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:29,067 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:29,112 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:29,114 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-07-22T12:28:29,136 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-07-22T12:28:29,136 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-07-22T12:28:29,137 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:28:29,148 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:28:29,150 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:28:29,622 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store 2023-07-22T12:28:30,048 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-07-22T12:28:30,048 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:30,049 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:28:30,049 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:28:30,049 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:28:30,049 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:28:30,049 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:28:30,049 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:28:30,049 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:28:30,051 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/.initializing 2023-07-22T12:28:30,052 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/WALs/jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:30,062 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:28:30,077 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43215%2C1690028905806, suffix=, logDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/WALs/jenkins-hbase3.apache.org,43215,1690028905806, archiveDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/oldWALs, maxLogs=10 2023-07-22T12:28:30,105 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/WALs/jenkins-hbase3.apache.org,43215,1690028905806/jenkins-hbase3.apache.org%2C43215%2C1690028905806.1690028910084, exclude list is [], retry=0 2023-07-22T12:28:30,133 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:28:30,140 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-07-22T12:28:30,218 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/WALs/jenkins-hbase3.apache.org,43215,1690028905806/jenkins-hbase3.apache.org%2C43215%2C1690028905806.1690028910084 2023-07-22T12:28:30,218 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:28:30,219 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:30,219 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:30,224 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,225 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,332 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-07-22T12:28:30,337 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:30,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:30,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-07-22T12:28:30,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:30,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:30,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-07-22T12:28:30,347 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:30,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:30,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-07-22T12:28:30,350 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:30,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:30,356 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,357 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,372 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-07-22T12:28:30,375 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-07-22T12:28:30,379 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:30,380 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9606651040, jitterRate=-0.10531090199947357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-07-22T12:28:30,386 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:28:30,387 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-07-22T12:28:30,421 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e0e8160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:30,476 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-07-22T12:28:30,498 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-07-22T12:28:30,498 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-07-22T12:28:30,502 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-07-22T12:28:30,503 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-07-22T12:28:30,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-07-22T12:28:30,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-07-22T12:28:30,547 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-07-22T12:28:30,566 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-07-22T12:28:30,569 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-07-22T12:28:30,573 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-07-22T12:28:30,575 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-07-22T12:28:30,576 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-07-22T12:28:30,579 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-07-22T12:28:30,587 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-07-22T12:28:30,588 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-07-22T12:28:30,589 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-07-22T12:28:30,591 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-07-22T12:28:30,607 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-07-22T12:28:30,608 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-07-22T12:28:30,614 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:28:30,614 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-07-22T12:28:30,614 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,614 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,615 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,43215,1690028905806, sessionid=0x100005bf1e10000, setting cluster-up flag (Was=false) 2023-07-22T12:28:30,624 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-07-22T12:28:30,632 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,632 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,637 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-07-22T12:28:30,638 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:30,642 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,642 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,653 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-07-22T12:28:30,654 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:30,659 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,659 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:30,664 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-07-22T12:28:30,665 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:30,686 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:36719 2023-07-22T12:28:30,687 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(997): ClusterId : 9bfdd9fa-516f-4b07-8560-00861e2f3f2f 2023-07-22T12:28:30,689 INFO [RS:0;jenkins-hbase3:36719 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-07-22T12:28:30,692 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-07-22T12:28:30,697 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-07-22T12:28:30,697 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-07-22T12:28:30,700 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-07-22T12:28:30,700 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-07-22T12:28:30,704 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-07-22T12:28:30,705 DEBUG [RS:0;jenkins-hbase3:36719 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3336ef0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:30,707 DEBUG [RS:0;jenkins-hbase3:36719 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32943a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:28:30,712 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-07-22T12:28:30,712 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-07-22T12:28:30,712 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-07-22T12:28:30,715 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,43215,1690028905806 with isa=jenkins-hbase3.apache.org/172.31.12.81:36719, startcode=1690028908802 2023-07-22T12:28:30,734 DEBUG [RS:0;jenkins-hbase3:36719 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:28:30,833 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:53993, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:28:30,841 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-07-22T12:28:30,840 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-07-22T12:28:30,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-07-22T12:28:30,860 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-07-22T12:28:30,871 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-07-22T12:28:30,871 WARN [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-07-22T12:28:30,871 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,43215,1690028905806 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-07-22T12:28:30,877 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:28:30,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:28:30,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:28:30,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-07-22T12:28:30,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-07-22T12:28:30,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:30,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:28:30,879 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:30,882 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1690028940882 2023-07-22T12:28:30,887 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-07-22T12:28:30,890 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:28:30,891 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-07-22T12:28:30,892 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-07-22T12:28:30,895 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:30,896 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:28:30,908 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-07-22T12:28:30,909 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-07-22T12:28:30,911 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-07-22T12:28:30,911 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-07-22T12:28:30,911 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-07-22T12:28:30,912 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:30,912 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-07-22T12:28:30,913 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:30,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-07-22T12:28:30,920 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-07-22T12:28:30,921 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-07-22T12:28:30,932 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7296d230 to 127.0.0.1:64849 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:28:30,941 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17b93e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:30,941 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-07-22T12:28:30,945 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-07-22T12:28:30,946 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-07-22T12:28:30,948 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690028910948,5,FailOnTimeoutGroup] 2023-07-22T12:28:30,948 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690028910948,5,FailOnTimeoutGroup] 2023-07-22T12:28:30,949 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:30,949 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-07-22T12:28:30,951 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:30,951 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:30,972 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,43215,1690028905806 with isa=jenkins-hbase3.apache.org/172.31.12.81:36719, startcode=1690028908802 2023-07-22T12:28:30,977 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43215 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:30,987 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:30,988 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:42225 2023-07-22T12:28:30,988 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=41475 2023-07-22T12:28:30,996 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-07-22T12:28:30,997 DEBUG [RS:0;jenkins-hbase3:36719 {}] zookeeper.ZKUtil(111): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:30,997 WARN [RS:0;jenkins-hbase3:36719 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-07-22T12:28:30,997 INFO [RS:0;jenkins-hbase3:36719 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:28:30,998 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:30,999 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,36719,1690028908802] 2023-07-22T12:28:31,019 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-07-22T12:28:31,029 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-07-22T12:28:31,043 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-07-22T12:28:31,047 INFO [RS:0;jenkins-hbase3:36719 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-07-22T12:28:31,047 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,048 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-07-22T12:28:31,057 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-07-22T12:28:31,058 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,059 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,059 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,059 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,059 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-07-22T12:28:31,059 DEBUG [RS:0;jenkins-hbase3:36719 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-07-22T12:28:31,060 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,060 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,061 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,061 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,061 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36719,1690028908802-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:28:31,079 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-07-22T12:28:31,083 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36719,1690028908802-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:31,099 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,36719,1690028908802 started 2023-07-22T12:28:31,099 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,36719,1690028908802, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:36719, sessionid=0x100005bf1e10001 2023-07-22T12:28:31,099 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-07-22T12:28:31,099 DEBUG [RS:0;jenkins-hbase3:36719 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:31,099 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36719,1690028908802' 2023-07-22T12:28:31,099 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-07-22T12:28:31,100 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-07-22T12:28:31,101 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-07-22T12:28:31,101 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-07-22T12:28:31,101 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36719,1690028908802' 2023-07-22T12:28:31,101 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:28:31,102 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:28:31,102 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-07-22T12:28:31,102 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-07-22T12:28:31,102 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-07-22T12:28:31,102 DEBUG [RS:0;jenkins-hbase3:36719 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:31,103 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36719,1690028908802' 2023-07-22T12:28:31,103 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-07-22T12:28:31,103 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-07-22T12:28:31,103 DEBUG [RS:0;jenkins-hbase3:36719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-07-22T12:28:31,104 INFO [RS:0;jenkins-hbase3:36719 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-07-22T12:28:31,104 INFO [RS:0;jenkins-hbase3:36719 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-07-22T12:28:31,214 INFO [RS:0;jenkins-hbase3:36719 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-07-22T12:28:31,217 INFO [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36719%2C1690028908802, suffix=, logDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802, archiveDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/oldWALs, maxLogs=32 2023-07-22T12:28:31,229 DEBUG [RS:0;jenkins-hbase3:36719 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028911218, exclude list is [], retry=0 2023-07-22T12:28:31,235 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:28:31,239 INFO [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028911218 2023-07-22T12:28:31,239 DEBUG [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:28:31,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:31,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:28:31,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:28:31,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:31,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:31,337 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:28:31,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:28:31,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:31,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:31,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:28:31,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:28:31,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:31,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:31,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740 2023-07-22T12:28:31,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740 2023-07-22T12:28:31,349 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:28:31,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:28:31,355 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:31,355 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11993455840, jitterRate=0.11697761714458466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:28:31,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:28:31,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:28:31,357 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:28:31,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:28:31,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:28:31,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:28:31,359 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:28:31,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:28:31,365 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-07-22T12:28:31,365 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-07-22T12:28:31,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-07-22T12:28:31,387 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-07-22T12:28:31,391 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-07-22T12:28:31,543 DEBUG [jenkins-hbase3:43215 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-07-22T12:28:31,551 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:31,557 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36719,1690028908802, state=OPENING 2023-07-22T12:28:31,566 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-07-22T12:28:31,571 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:31,571 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:31,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:28:31,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:28:31,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:31,768 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:31,771 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=AdminService, sasl=false 2023-07-22T12:28:31,774 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36976, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-07-22T12:28:31,787 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-07-22T12:28:31,788 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-07-22T12:28:31,789 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-07-22T12:28:31,792 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta, suffix=.meta, logDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802, archiveDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/oldWALs, maxLogs=32 2023-07-22T12:28:31,805 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028911793.meta, exclude list is [], retry=0 2023-07-22T12:28:31,809 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:28:31,812 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028911793.meta 2023-07-22T12:28:31,813 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:28:31,813 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:31,818 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:31,819 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-07-22T12:28:31,835 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-07-22T12:28:31,836 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-07-22T12:28:31,840 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-07-22T12:28:31,841 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:31,841 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-07-22T12:28:31,841 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-07-22T12:28:31,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-07-22T12:28:31,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-07-22T12:28:31,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:31,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:31,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-07-22T12:28:31,847 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-07-22T12:28:31,848 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:31,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:31,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-07-22T12:28:31,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-07-22T12:28:31,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:31,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-07-22T12:28:31,853 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740 2023-07-22T12:28:31,855 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740 2023-07-22T12:28:31,858 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-07-22T12:28:31,860 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-07-22T12:28:31,862 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10892492160, jitterRate=0.014442384243011475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-07-22T12:28:31,863 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-07-22T12:28:31,876 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1690028911759 2023-07-22T12:28:31,897 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-07-22T12:28:31,898 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-07-22T12:28:31,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:31,901 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36719,1690028908802, state=OPEN 2023-07-22T12:28:31,903 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:28:31,903 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-07-22T12:28:31,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:28:31,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-07-22T12:28:31,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-07-22T12:28:31,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36719,1690028908802 in 326 msec 2023-07-22T12:28:31,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-07-22T12:28:31,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 535 msec 2023-07-22T12:28:31,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.1900 sec 2023-07-22T12:28:31,921 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1690028911921, completionTime=-1 2023-07-22T12:28:31,921 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-07-22T12:28:31,922 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-07-22T12:28:32,114 DEBUG [hconnection-0x52854ce7-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:32,118 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36978, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:32,133 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-07-22T12:28:32,134 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1690028972134 2023-07-22T12:28:32,134 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1690029032134 2023-07-22T12:28:32,134 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 212 msec 2023-07-22T12:28:32,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43215,1690028905806-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:32,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43215,1690028905806-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:32,157 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43215,1690028905806-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:32,159 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:43215, period=300000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:32,160 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-07-22T12:28:32,166 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-07-22T12:28:32,182 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-07-22T12:28:32,184 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-07-22T12:28:32,192 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-07-22T12:28:32,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:32,198 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:32,199 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:32,625 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2cc0bf07e1b2d5a0dc35be6598786b24, NAME => 'hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:33,038 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:33,038 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 2cc0bf07e1b2d5a0dc35be6598786b24, disabling compactions & flushes 2023-07-22T12:28:33,038 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,038 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,038 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. after waiting 0 ms 2023-07-22T12:28:33,038 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,038 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,038 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2cc0bf07e1b2d5a0dc35be6598786b24: 2023-07-22T12:28:33,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:33,054 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1690028913042"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028913042"}]},"ts":"1690028913042"} 2023-07-22T12:28:33,085 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:33,087 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:33,092 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028913087"}]},"ts":"1690028913087"} 2023-07-22T12:28:33,098 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-07-22T12:28:33,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=2cc0bf07e1b2d5a0dc35be6598786b24, ASSIGN}] 2023-07-22T12:28:33,105 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=2cc0bf07e1b2d5a0dc35be6598786b24, ASSIGN 2023-07-22T12:28:33,107 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=2cc0bf07e1b2d5a0dc35be6598786b24, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:33,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=2cc0bf07e1b2d5a0dc35be6598786b24, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:33,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 2cc0bf07e1b2d5a0dc35be6598786b24, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:33,420 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,421 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 2cc0bf07e1b2d5a0dc35be6598786b24, NAME => 'hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:33,422 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:33,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:33,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,423 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,425 INFO [StoreOpener-2cc0bf07e1b2d5a0dc35be6598786b24-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,427 INFO [StoreOpener-2cc0bf07e1b2d5a0dc35be6598786b24-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc0bf07e1b2d5a0dc35be6598786b24 columnFamilyName info 2023-07-22T12:28:33,427 DEBUG [StoreOpener-2cc0bf07e1b2d5a0dc35be6598786b24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:33,428 INFO [StoreOpener-2cc0bf07e1b2d5a0dc35be6598786b24-1 {}] regionserver.HStore(324): Store=2cc0bf07e1b2d5a0dc35be6598786b24/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:33,429 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,430 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,433 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:28:33,437 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:33,438 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 2cc0bf07e1b2d5a0dc35be6598786b24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9594203360, jitterRate=-0.10647018253803253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:28:33,439 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 2cc0bf07e1b2d5a0dc35be6598786b24: 2023-07-22T12:28:33,441 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24., pid=6, masterSystemTime=1690028913415 2023-07-22T12:28:33,444 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,444 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:28:33,445 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=2cc0bf07e1b2d5a0dc35be6598786b24, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:33,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-07-22T12:28:33,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 2cc0bf07e1b2d5a0dc35be6598786b24, server=jenkins-hbase3.apache.org,36719,1690028908802 in 188 msec 2023-07-22T12:28:33,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-07-22T12:28:33,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=2cc0bf07e1b2d5a0dc35be6598786b24, ASSIGN in 349 msec 2023-07-22T12:28:33,457 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:33,457 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028913457"}]},"ts":"1690028913457"} 2023-07-22T12:28:33,461 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-07-22T12:28:33,464 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:33,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.2800 sec 2023-07-22T12:28:33,497 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-07-22T12:28:33,498 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:33,498 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36719-0x100005bf1e10001, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:33,498 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-07-22T12:28:33,545 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-07-22T12:28:33,562 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:33,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 31 msec 2023-07-22T12:28:33,579 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-07-22T12:28:33,590 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:33,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 15 msec 2023-07-22T12:28:33,606 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-07-22T12:28:33,609 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-07-22T12:28:33,609 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.687sec 2023-07-22T12:28:33,612 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-07-22T12:28:33,614 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-07-22T12:28:33,616 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-07-22T12:28:33,617 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-07-22T12:28:33,617 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-07-22T12:28:33,619 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43215,1690028905806-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-07-22T12:28:33,620 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43215,1690028905806-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-07-22T12:28:33,639 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-07-22T12:28:33,641 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-07-22T12:28:33,652 DEBUG [Listener at localhost/35041 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x38e16d3f to 127.0.0.1:64849 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:28:33,660 DEBUG [Listener at localhost/35041 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79556cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:33,667 DEBUG [Listener at localhost/35041 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-07-22T12:28:33,667 DEBUG [Listener at localhost/35041 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-07-22T12:28:33,685 DEBUG [hconnection-0x2efc7c1-shared-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:33,695 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36982, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/test.cache.data in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.tmp.dir in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.log.dir in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/mapreduce.cluster.local.dir in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-07-22T12:28:33,706 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/nfs.dump.dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.journalnode.edits.dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-07-22T12:28:33,707 INFO [Listener at localhost/35041 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-07-22T12:28:35,154 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:28:35,521 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:28:35,527 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39125_jobhistory____bc3bo8/webapp 2023-07-22T12:28:35,703 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-07-22T12:28:35,704 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-07-22T12:28:35,704 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:28:35,708 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:28:35,816 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:35,935 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:35,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-07-22T12:28:35,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:35,976 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-07-22T12:28:35,976 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:35,978 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:28:35,978 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-07-22T12:28:35,978 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:28:35,978 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-07-22T12:28:35,978 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-07-22T12:28:35,978 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:36,199 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-07-22T12:28:36,228 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39125 2023-07-22T12:28:36,651 WARN [Listener at jenkins-hbase3.apache.org/41221 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:28:36,728 WARN [Listener at jenkins-hbase3.apache.org/41221 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:28:36,728 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:28:36,734 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_35847_cluster____h4pnxm/webapp 2023-07-22T12:28:36,801 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:28:36,802 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-07-22T12:28:36,802 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:28:36,802 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:28:36,810 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,021 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,099 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,107 INFO [Listener at jenkins-hbase3.apache.org/41221 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:35847 2023-07-22T12:28:37,377 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:28:37,456 WARN [Listener at jenkins-hbase3.apache.org/39907 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:28:37,457 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-07-22T12:28:37,458 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-07-22T12:28:37,479 WARN [Listener at jenkins-hbase3.apache.org/39907 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:28:37,480 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:28:37,488 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45407_node____d0i2wp/webapp 2023-07-22T12:28:37,540 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:28:37,541 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:28:37,541 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:28:37,541 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:28:37,548 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,571 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,676 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,686 INFO [Listener at jenkins-hbase3.apache.org/39907 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45407 2023-07-22T12:28:37,856 WARN [Listener at jenkins-hbase3.apache.org/36493 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-07-22T12:28:37,863 WARN [Listener at jenkins-hbase3.apache.org/36493 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-07-22T12:28:37,863 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): jetty-6.1.26 2023-07-22T12:28:37,872 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37811_node____8c0ln7/webapp 2023-07-22T12:28:37,928 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-07-22T12:28:37,928 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-07-22T12:28:37,928 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-07-22T12:28:37,929 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-07-22T12:28:37,940 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:37,965 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:38,045 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-07-22T12:28:38,053 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37811 2023-07-22T12:28:38,084 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-07-22T12:28:38,084 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:42225/backupUT 2023-07-22T12:28:38,096 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:28:38,103 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45564, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:28:38,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-07-22T12:28:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-07-22T12:28:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:28:38,144 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:38,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 34 msec 2023-07-22T12:28:38,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-07-22T12:28:38,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-07-22T12:28:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-07-22T12:28:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:28:38,245 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:38,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 16 msec 2023-07-22T12:28:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-07-22T12:28:38,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-07-22T12:28:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-07-22T12:28:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:28:38,355 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:38,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 15 msec 2023-07-22T12:28:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-07-22T12:28:38,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-07-22T12:28:38,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-07-22T12:28:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:28:38,463 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:38,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 15 msec 2023-07-22T12:28:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-07-22T12:28:38,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1690028918084', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1690028918084 2023-07-22T12:28:38,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1690028918084 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:38,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1690028918084" procId is: 13 2023-07-22T12:28:38,577 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:38,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1690028918084 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:38,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:28:38,607 INFO [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d465cc75c102ecbb2abf7437d3ee9ec0, NAME => 'test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1690028918084', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:38,622 DEBUG [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(888): Instantiated test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:38,622 DEBUG [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1668): Closing d465cc75c102ecbb2abf7437d3ee9ec0, disabling compactions & flushes 2023-07-22T12:28:38,622 INFO [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1690): Closing region test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,622 DEBUG [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,622 DEBUG [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. after waiting 0 ms 2023-07-22T12:28:38,622 DEBUG [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,622 INFO [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1909): Closed test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,622 DEBUG [RegionOpenAndInit-test-1690028918084-pool-0 {}] regionserver.HRegion(1622): Region close journal for d465cc75c102ecbb2abf7437d3ee9ec0: 2023-07-22T12:28:38,623 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1690028918084 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:38,624 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1690028918623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028918623"}]},"ts":"1690028918623"} 2023-07-22T12:28:38,626 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:38,627 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1690028918084 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:38,627 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690028918084","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028918627"}]},"ts":"1690028918627"} 2023-07-22T12:28:38,629 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690028918084, state=ENABLING in hbase:meta 2023-07-22T12:28:38,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690028918084, region=d465cc75c102ecbb2abf7437d3ee9ec0, ASSIGN}] 2023-07-22T12:28:38,636 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1690028918084, region=d465cc75c102ecbb2abf7437d3ee9ec0, ASSIGN 2023-07-22T12:28:38,637 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1690028918084, region=d465cc75c102ecbb2abf7437d3ee9ec0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:28:38,789 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=d465cc75c102ecbb2abf7437d3ee9ec0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:38,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure d465cc75c102ecbb2abf7437d3ee9ec0, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:28:38,953 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,953 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => d465cc75c102ecbb2abf7437d3ee9ec0, NAME => 'test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:38,954 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:38,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1690028918084 d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:38,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,954 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,956 INFO [StoreOpener-d465cc75c102ecbb2abf7437d3ee9ec0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,959 INFO [StoreOpener-d465cc75c102ecbb2abf7437d3ee9ec0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d465cc75c102ecbb2abf7437d3ee9ec0 columnFamilyName f 2023-07-22T12:28:38,959 DEBUG [StoreOpener-d465cc75c102ecbb2abf7437d3ee9ec0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:38,960 INFO [StoreOpener-d465cc75c102ecbb2abf7437d3ee9ec0-1 {}] regionserver.HStore(324): Store=d465cc75c102ecbb2abf7437d3ee9ec0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:38,962 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,962 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,966 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:28:38,972 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:38,973 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened d465cc75c102ecbb2abf7437d3ee9ec0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10919218400, jitterRate=0.016931459307670593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:28:38,974 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for d465cc75c102ecbb2abf7437d3ee9ec0: 2023-07-22T12:28:38,975 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0., pid=15, masterSystemTime=1690028918944 2023-07-22T12:28:38,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,978 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:28:38,979 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=d465cc75c102ecbb2abf7437d3ee9ec0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:38,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-07-22T12:28:38,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure d465cc75c102ecbb2abf7437d3ee9ec0, server=jenkins-hbase3.apache.org,36719,1690028908802 in 191 msec 2023-07-22T12:28:38,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-07-22T12:28:38,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1690028918084, region=d465cc75c102ecbb2abf7437d3ee9ec0, ASSIGN in 351 msec 2023-07-22T12:28:38,989 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1690028918084 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:38,989 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1690028918084","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028918989"}]},"ts":"1690028918989"} 2023-07-22T12:28:38,991 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1690028918084, state=ENABLED in hbase:meta 2023-07-22T12:28:38,994 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1690028918084 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:38,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1690028918084 in 422 msec 2023-07-22T12:28:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-07-22T12:28:39,188 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1690028918084, procId: 13 completed 2023-07-22T12:28:39,190 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x429232f5 to 127.0.0.1:64849 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-07-22T12:28:39,198 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53058fab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:39,202 DEBUG [hconnection-0x35089fd7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:39,208 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36992, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:39,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36719 {}] regionserver.HRegion(8210): writing data to region test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:28:39,322 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16900289180841', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16900289180841 2023-07-22T12:28:39,326 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900289180841 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:39,326 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:39,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16900289180841" procId is: 16 2023-07-22T12:28:39,328 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16900289180841 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:28:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:28:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:28:39,746 INFO [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 62c1d4e2655165c29f9a13c43a911f4c, NAME => 'ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16900289180841', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:28:40,163 DEBUG [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:40,163 DEBUG [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1668): Closing 62c1d4e2655165c29f9a13c43a911f4c, disabling compactions & flushes 2023-07-22T12:28:40,163 INFO [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,163 DEBUG [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,163 DEBUG [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. after waiting 0 ms 2023-07-22T12:28:40,163 DEBUG [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,164 INFO [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,164 DEBUG [RegionOpenAndInit-ns2:test-16900289180841-pool-0 {}] regionserver.HRegion(1622): Region close journal for 62c1d4e2655165c29f9a13c43a911f4c: 2023-07-22T12:28:40,165 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16900289180841 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:40,166 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690028920166"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028920166"}]},"ts":"1690028920166"} 2023-07-22T12:28:40,169 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:40,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16900289180841 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:40,170 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900289180841","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028920170"}]},"ts":"1690028920170"} 2023-07-22T12:28:40,173 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900289180841, state=ENABLING in hbase:meta 2023-07-22T12:28:40,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900289180841, region=62c1d4e2655165c29f9a13c43a911f4c, ASSIGN}] 2023-07-22T12:28:40,180 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16900289180841, region=62c1d4e2655165c29f9a13c43a911f4c, ASSIGN 2023-07-22T12:28:40,181 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16900289180841, region=62c1d4e2655165c29f9a13c43a911f4c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:40,332 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=62c1d4e2655165c29f9a13c43a911f4c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:40,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 62c1d4e2655165c29f9a13c43a911f4c, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:28:40,492 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 62c1d4e2655165c29f9a13c43a911f4c, NAME => 'ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:40,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:40,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900289180841 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:40,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,495 INFO [StoreOpener-62c1d4e2655165c29f9a13c43a911f4c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,497 INFO [StoreOpener-62c1d4e2655165c29f9a13c43a911f4c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62c1d4e2655165c29f9a13c43a911f4c columnFamilyName f 2023-07-22T12:28:40,497 DEBUG [StoreOpener-62c1d4e2655165c29f9a13c43a911f4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:40,498 INFO [StoreOpener-62c1d4e2655165c29f9a13c43a911f4c-1 {}] regionserver.HStore(324): Store=62c1d4e2655165c29f9a13c43a911f4c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:40,499 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,500 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:28:40,511 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:40,512 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 62c1d4e2655165c29f9a13c43a911f4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9794719360, jitterRate=-0.0877956748008728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:28:40,513 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 62c1d4e2655165c29f9a13c43a911f4c: 2023-07-22T12:28:40,514 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c., pid=18, masterSystemTime=1690028920488 2023-07-22T12:28:40,516 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,517 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:28:40,517 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=62c1d4e2655165c29f9a13c43a911f4c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:40,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-07-22T12:28:40,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 62c1d4e2655165c29f9a13c43a911f4c, server=jenkins-hbase3.apache.org,36719,1690028908802 in 185 msec 2023-07-22T12:28:40,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-07-22T12:28:40,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16900289180841, region=62c1d4e2655165c29f9a13c43a911f4c, ASSIGN in 345 msec 2023-07-22T12:28:40,526 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16900289180841 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:40,526 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16900289180841","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028920526"}]},"ts":"1690028920526"} 2023-07-22T12:28:40,528 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16900289180841, state=ENABLED in hbase:meta 2023-07-22T12:28:40,531 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16900289180841 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:40,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16900289180841 in 1.2090 sec 2023-07-22T12:28:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-07-22T12:28:41,439 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16900289180841, procId: 16 completed 2023-07-22T12:28:41,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36719 {}] regionserver.HRegion(8210): writing data to region ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. with WAL disabled. Data may be lost in the event of a crash. 2023-07-22T12:28:41,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16900289180842', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16900289180842 2023-07-22T12:28:41,529 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900289180842 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:41,529 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:41,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16900289180842" procId is: 19 2023-07-22T12:28:41,530 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16900289180842 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:28:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:28:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:28:41,946 INFO [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d5a34dd2c7434d800baac09500ad347f, NAME => 'ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16900289180842', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:28:42,362 DEBUG [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:42,363 DEBUG [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1668): Closing d5a34dd2c7434d800baac09500ad347f, disabling compactions & flushes 2023-07-22T12:28:42,363 INFO [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,363 DEBUG [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,363 DEBUG [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. after waiting 0 ms 2023-07-22T12:28:42,363 DEBUG [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,363 INFO [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,363 DEBUG [RegionOpenAndInit-ns3:test-16900289180842-pool-0 {}] regionserver.HRegion(1622): Region close journal for d5a34dd2c7434d800baac09500ad347f: 2023-07-22T12:28:42,364 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16900289180842 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:42,365 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690028922364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028922364"}]},"ts":"1690028922364"} 2023-07-22T12:28:42,367 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:42,368 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16900289180842 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:42,369 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900289180842","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028922368"}]},"ts":"1690028922368"} 2023-07-22T12:28:42,370 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900289180842, state=ENABLING in hbase:meta 2023-07-22T12:28:42,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900289180842, region=d5a34dd2c7434d800baac09500ad347f, ASSIGN}] 2023-07-22T12:28:42,376 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16900289180842, region=d5a34dd2c7434d800baac09500ad347f, ASSIGN 2023-07-22T12:28:42,377 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16900289180842, region=d5a34dd2c7434d800baac09500ad347f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:42,529 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=d5a34dd2c7434d800baac09500ad347f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:42,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure d5a34dd2c7434d800baac09500ad347f, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:28:42,689 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => d5a34dd2c7434d800baac09500ad347f, NAME => 'ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:42,689 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:42,690 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900289180842 d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,690 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:42,690 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,690 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,694 INFO [StoreOpener-d5a34dd2c7434d800baac09500ad347f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,696 INFO [StoreOpener-d5a34dd2c7434d800baac09500ad347f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d5a34dd2c7434d800baac09500ad347f columnFamilyName f 2023-07-22T12:28:42,696 DEBUG [StoreOpener-d5a34dd2c7434d800baac09500ad347f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:42,697 INFO [StoreOpener-d5a34dd2c7434d800baac09500ad347f-1 {}] regionserver.HStore(324): Store=d5a34dd2c7434d800baac09500ad347f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:42,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns3/test-16900289180842/d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns3/test-16900289180842/d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,702 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:28:42,705 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns3/test-16900289180842/d5a34dd2c7434d800baac09500ad347f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:42,706 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened d5a34dd2c7434d800baac09500ad347f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10893956960, jitterRate=0.01457880437374115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:28:42,706 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for d5a34dd2c7434d800baac09500ad347f: 2023-07-22T12:28:42,707 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f., pid=21, masterSystemTime=1690028922684 2023-07-22T12:28:42,710 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,710 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:28:42,710 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=d5a34dd2c7434d800baac09500ad347f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:42,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-07-22T12:28:42,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure d5a34dd2c7434d800baac09500ad347f, server=jenkins-hbase3.apache.org,36719,1690028908802 in 181 msec 2023-07-22T12:28:42,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-07-22T12:28:42,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16900289180842, region=d5a34dd2c7434d800baac09500ad347f, ASSIGN in 341 msec 2023-07-22T12:28:42,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16900289180842 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:42,718 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16900289180842","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028922718"}]},"ts":"1690028922718"} 2023-07-22T12:28:42,720 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16900289180842, state=ENABLED in hbase:meta 2023-07-22T12:28:42,722 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16900289180842 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:42,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16900289180842 in 1.1970 sec 2023-07-22T12:28:43,460 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16900289180842' 2023-07-22T12:28:43,460 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1690028918084' 2023-07-22T12:28:43,461 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16900289180841' 2023-07-22T12:28:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-07-22T12:28:43,639 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16900289180842, procId: 19 completed 2023-07-22T12:28:43,639 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16900289180842 get assigned. Timeout = 60000ms 2023-07-22T12:28:43,641 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:28:43,646 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16900289180842 assigned to meta. Checking AM states. 2023-07-22T12:28:43,647 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:28:43,647 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16900289180842 assigned. 2023-07-22T12:28:43,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16900289180843', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16900289180843 2023-07-22T12:28:43,652 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900289180843 execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:43,652 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:43,653 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16900289180843" procId is: 22 2023-07-22T12:28:43,653 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16900289180843 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:28:43,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:28:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:28:44,067 INFO [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0966504315f369581de1e5b13d64f65b, NAME => 'ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16900289180843', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:44,084 DEBUG [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:44,084 DEBUG [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1668): Closing 0966504315f369581de1e5b13d64f65b, disabling compactions & flushes 2023-07-22T12:28:44,084 INFO [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,084 DEBUG [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,084 DEBUG [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. after waiting 0 ms 2023-07-22T12:28:44,084 DEBUG [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,084 INFO [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,084 DEBUG [RegionOpenAndInit-ns4:test-16900289180843-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0966504315f369581de1e5b13d64f65b: 2023-07-22T12:28:44,086 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16900289180843 execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:44,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1690028924086"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028924086"}]},"ts":"1690028924086"} 2023-07-22T12:28:44,088 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:44,089 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16900289180843 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:44,090 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900289180843","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028924089"}]},"ts":"1690028924089"} 2023-07-22T12:28:44,091 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900289180843, state=ENABLING in hbase:meta 2023-07-22T12:28:44,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900289180843, region=0966504315f369581de1e5b13d64f65b, ASSIGN}] 2023-07-22T12:28:44,102 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16900289180843, region=0966504315f369581de1e5b13d64f65b, ASSIGN 2023-07-22T12:28:44,103 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16900289180843, region=0966504315f369581de1e5b13d64f65b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:44,255 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=0966504315f369581de1e5b13d64f65b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:44,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 0966504315f369581de1e5b13d64f65b, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:28:44,415 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,415 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 0966504315f369581de1e5b13d64f65b, NAME => 'ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:44,416 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:44,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16900289180843 0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:44,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,418 INFO [StoreOpener-0966504315f369581de1e5b13d64f65b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,420 INFO [StoreOpener-0966504315f369581de1e5b13d64f65b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0966504315f369581de1e5b13d64f65b columnFamilyName f 2023-07-22T12:28:44,420 DEBUG [StoreOpener-0966504315f369581de1e5b13d64f65b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:44,420 INFO [StoreOpener-0966504315f369581de1e5b13d64f65b-1 {}] regionserver.HStore(324): Store=0966504315f369581de1e5b13d64f65b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:44,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns4/test-16900289180843/0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns4/test-16900289180843/0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,425 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 0966504315f369581de1e5b13d64f65b 2023-07-22T12:28:44,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns4/test-16900289180843/0966504315f369581de1e5b13d64f65b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:44,429 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 0966504315f369581de1e5b13d64f65b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9541002880, jitterRate=-0.11142486333847046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-07-22T12:28:44,430 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 0966504315f369581de1e5b13d64f65b: 2023-07-22T12:28:44,431 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b., pid=24, masterSystemTime=1690028924410 2023-07-22T12:28:44,433 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,433 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:28:44,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=0966504315f369581de1e5b13d64f65b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:44,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-07-22T12:28:44,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 0966504315f369581de1e5b13d64f65b, server=jenkins-hbase3.apache.org,36719,1690028908802 in 179 msec 2023-07-22T12:28:44,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-07-22T12:28:44,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16900289180843, region=0966504315f369581de1e5b13d64f65b, ASSIGN in 338 msec 2023-07-22T12:28:44,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16900289180843 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:44,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16900289180843","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028924442"}]},"ts":"1690028924442"} 2023-07-22T12:28:44,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16900289180843, state=ENABLED in hbase:meta 2023-07-22T12:28:44,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16900289180843 execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:44,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16900289180843 in 798 msec 2023-07-22T12:28:44,462 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:28:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-07-22T12:28:44,762 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16900289180843, procId: 22 completed 2023-07-22T12:28:44,762 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16900289180843 get assigned. Timeout = 60000ms 2023-07-22T12:28:44,762 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:28:44,771 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16900289180843 assigned to meta. Checking AM states. 2023-07-22T12:28:44,771 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-07-22T12:28:44,771 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16900289180843 assigned. 2023-07-22T12:28:44,772 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x429232f5 to 127.0.0.1:64849 2023-07-22T12:28:44,772 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:44,819 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.ResourceChecker(147): before: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=156, ProcessCount=169, AvailableMemoryMB=16980 2023-07-22T12:28:44,819 WARN [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-07-22T12:28:44,823 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupRepair(57): Running stage 0 2023-07-22T12:28:44,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-07-22T12:28:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-07-22T12:28:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:28:44,856 DEBUG [Listener at localhost/35041-EventThread {}] zookeeper.ZKWatcher(604): master:43215-0x100005bf1e10000, quorum=127.0.0.1:64849, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-07-22T12:28:44,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 20 msec 2023-07-22T12:28:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-07-22T12:28:44,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-07-22T12:28:44,970 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:44,970 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:44,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-07-22T12:28:44,971 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:28:44,997 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:45,015 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:45,016 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing ccd715104b6c80cc2e57c916da0834fd, disabling compactions & flushes 2023-07-22T12:28:45,016 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,016 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,016 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. after waiting 0 ms 2023-07-22T12:28:45,016 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,016 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,016 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:45,018 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:45,018 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028925018"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028925018"}]},"ts":"1690028925018"} 2023-07-22T12:28:45,020 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:45,021 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:45,022 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028925022"}]},"ts":"1690028925022"} 2023-07-22T12:28:45,024 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:28:45,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN}] 2023-07-22T12:28:45,030 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN 2023-07-22T12:28:45,031 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:28:45,183 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:45,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:28:45,344 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:45,345 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:45,345 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,345 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:45,345 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,345 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,350 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,353 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName meta 2023-07-22T12:28:45,354 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:45,354 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:45,355 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,357 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName session 2023-07-22T12:28:45,357 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:45,358 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:45,360 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:28:45,365 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:45,367 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:45,368 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened ccd715104b6c80cc2e57c916da0834fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11877593280, jitterRate=0.10618707537651062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:28:45,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:45,370 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., pid=28, masterSystemTime=1690028925338 2023-07-22T12:28:45,373 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,373 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:45,376 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:45,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-07-22T12:28:45,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 196 msec 2023-07-22T12:28:45,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-07-22T12:28:45,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN in 358 msec 2023-07-22T12:28:45,389 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:45,390 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028925389"}]},"ts":"1690028925389"} 2023-07-22T12:28:45,392 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:28:45,396 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:45,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 431 msec 2023-07-22T12:28:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-07-22T12:28:45,578 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-07-22T12:28:45,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-07-22T12:28:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-07-22T12:28:45,600 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-07-22T12:28:45,600 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:45,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-07-22T12:28:45,602 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-07-22T12:28:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:28:45,614 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 6705f3d0b146a5036498f0b2faf38255, NAME => 'backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1 2023-07-22T12:28:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:28:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:28:45,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1690028918084 2023-07-22T12:28:45,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1690028918084 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:45,970 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:28:45,970 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:45,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900289180843 2023-07-22T12:28:45,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16900289180843 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:45,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900289180841 2023-07-22T12:28:45,972 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16900289180841 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:45,973 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900289180842 2023-07-22T12:28:45,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16900289180842 Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:46,028 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:46,028 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 6705f3d0b146a5036498f0b2faf38255, disabling compactions & flushes 2023-07-22T12:28:46,029 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,029 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,029 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. after waiting 0 ms 2023-07-22T12:28:46,029 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,029 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,029 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 6705f3d0b146a5036498f0b2faf38255: 2023-07-22T12:28:46,030 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-07-22T12:28:46,031 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1690028926030"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028926030"}]},"ts":"1690028926030"} 2023-07-22T12:28:46,033 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:46,034 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-07-22T12:28:46,034 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028926034"}]},"ts":"1690028926034"} 2023-07-22T12:28:46,036 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-07-22T12:28:46,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=6705f3d0b146a5036498f0b2faf38255, ASSIGN}] 2023-07-22T12:28:46,042 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=6705f3d0b146a5036498f0b2faf38255, ASSIGN 2023-07-22T12:28:46,043 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=6705f3d0b146a5036498f0b2faf38255, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36719,1690028908802; forceNewPlan=false, retain=false 2023-07-22T12:28:46,195 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=6705f3d0b146a5036498f0b2faf38255, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:46,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 6705f3d0b146a5036498f0b2faf38255, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:28:46,354 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,354 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 6705f3d0b146a5036498f0b2faf38255, NAME => 'backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:46,355 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:46,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:46,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,357 INFO [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,358 INFO [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6705f3d0b146a5036498f0b2faf38255 columnFamilyName meta 2023-07-22T12:28:46,359 DEBUG [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:46,359 INFO [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] regionserver.HStore(324): Store=6705f3d0b146a5036498f0b2faf38255/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:46,359 INFO [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,361 INFO [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6705f3d0b146a5036498f0b2faf38255 columnFamilyName session 2023-07-22T12:28:46,361 DEBUG [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:46,361 INFO [StoreOpener-6705f3d0b146a5036498f0b2faf38255-1 {}] regionserver.HStore(324): Store=6705f3d0b146a5036498f0b2faf38255/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:46,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system_bulk/6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system_bulk/6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:28:46,366 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:28:46,368 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system_bulk/6705f3d0b146a5036498f0b2faf38255/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-07-22T12:28:46,369 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 6705f3d0b146a5036498f0b2faf38255; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11978375520, jitterRate=0.11557315289974213}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:28:46,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 6705f3d0b146a5036498f0b2faf38255: 2023-07-22T12:28:46,370 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255., pid=31, masterSystemTime=1690028926349 2023-07-22T12:28:46,372 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,373 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:28:46,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=6705f3d0b146a5036498f0b2faf38255, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:46,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-07-22T12:28:46,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 6705f3d0b146a5036498f0b2faf38255, server=jenkins-hbase3.apache.org,36719,1690028908802 in 178 msec 2023-07-22T12:28:46,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-07-22T12:28:46,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=6705f3d0b146a5036498f0b2faf38255, ASSIGN in 337 msec 2023-07-22T12:28:46,380 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-07-22T12:28:46,380 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028926380"}]},"ts":"1690028926380"} 2023-07-22T12:28:46,382 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-07-22T12:28:46,385 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-07-22T12:28:46,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 788 msec 2023-07-22T12:28:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-07-22T12:28:46,709 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-07-22T12:28:46,762 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2599c3b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:46,763 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:28:46,764 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:28:46,770 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:46,773 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34680, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:46,842 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:28:46,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028926894 (current time:1690028926894). 2023-07-22T12:28:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:28:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:28:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ca7da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:46,908 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:34694, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:28:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:28:46,927 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:28:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:46,933 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:28:46,936 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:28:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:47,348 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:28:47,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd}] 2023-07-22T12:28:47,358 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:47,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-07-22T12:28:47,519 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:47,521 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=45 B heapSize=632 B 2023-07-22T12:28:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:47,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/8973d311c3014795b5c35b33ce8fd886 is 49, key is activesession:/session:c/1690028926876/Put/seqid=0 2023-07-22T12:28:48,018 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:48,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:48,094 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/8973d311c3014795b5c35b33ce8fd886 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:48,102 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, entries=1, sequenceid=5, filesize=4.9 K 2023-07-22T12:28:48,106 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 585ms, sequenceid=5, compaction requested=false 2023-07-22T12:28:48,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:28:48,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:48,108 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. for snapshot_backup_system completed. 2023-07-22T12:28:48,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:28:48,115 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:28:48,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:28:48,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886] hfiles 2023-07-22T12:28:48,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 for snapshot=snapshot_backup_system 2023-07-22T12:28:48,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:48,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-07-22T12:28:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-07-22T12:28:48,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:48,546 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:48,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-07-22T12:28:48,549 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:28:48,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd in 1.1960 sec 2023-07-22T12:28:48,551 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:28:48,555 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:28:48,555 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:28:48,557 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:28:48,983 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:28:48,995 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:28:48,996 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:28:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:49,833 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:28:49,833 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-07-22T12:28:49,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.9140 sec 2023-07-22T12:28:49,919 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:28:49,968 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-07-22T12:28:49,968 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16900289180843' 2023-07-22T12:28:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-07-22T12:28:51,038 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(115): Backup backup_1690028926808 started at 1690028931038. 2023-07-22T12:28:51,070 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(119): Backup session backup_1690028926808 has been started. Backup session finished. Status: FAILURE 2023-07-22T12:28:51,071 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:51,071 ERROR [Listener at jenkins-hbase3.apache.org/36493 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] ... 32 more 2023-07-22T12:28:51,080 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398debff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:51,086 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccc9714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:51,087 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:28:51,089 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59596, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:28:51,095 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:51,096 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47130, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1690028926808,startts=1690028931038,failedts=1690028931117,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1690028926808 2023-07-22T12:28:51,118 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:28:51,119 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:28:51,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:28:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:28:51,153 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028931153"}]},"ts":"1690028931153"} 2023-07-22T12:28:51,155 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:28:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:28:51,158 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:28:51,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN}] 2023-07-22T12:28:51,162 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN 2023-07-22T12:28:51,163 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:51,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:28:51,330 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(119): Close ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:51,330 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1668): Closing ccd715104b6c80cc2e57c916da0834fd, disabling compactions & flushes 2023-07-22T12:28:51,330 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1690): Closing region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:51,330 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:51,330 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. after waiting 0 ms 2023-07-22T12:28:51,330 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:51,330 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=214 B heapSize=800 B 2023-07-22T12:28:51,337 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/ded5dbf29e1c4c659ed9139c21ad4ef9 is 218, key is session:backup_1690028926808/session:context/1690028931068/Put/seqid=0 2023-07-22T12:28:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:28:51,747 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/ded5dbf29e1c4c659ed9139c21ad4ef9 2023-07-22T12:28:51,758 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ded5dbf29e1c4c659ed9139c21ad4ef9 2023-07-22T12:28:51,759 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/ded5dbf29e1c4c659ed9139c21ad4ef9 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/ded5dbf29e1c4c659ed9139c21ad4ef9 2023-07-22T12:28:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:28:51,767 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/ded5dbf29e1c4c659ed9139c21ad4ef9, entries=1, sequenceid=9, filesize=5.2 K 2023-07-22T12:28:51,770 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 439ms, sequenceid=9, compaction requested=false 2023-07-22T12:28:51,772 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:51,774 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ded5dbf29e1c4c659ed9139c21ad4ef9 2023-07-22T12:28:51,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-07-22T12:28:51,781 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:28:51,782 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1909): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:51,782 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1622): Region close journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:51,784 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(163): Closed ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:51,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSED 2023-07-22T12:28:51,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=36, resume processing ppid=35 2023-07-22T12:28:51,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, ppid=35, state=SUCCESS; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 622 msec 2023-07-22T12:28:51,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-07-22T12:28:51,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN in 629 msec 2023-07-22T12:28:51,793 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028931793"}]},"ts":"1690028931793"} 2023-07-22T12:28:51,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:28:51,797 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:28:51,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; DisableTableProcedure table=backup:system in 655 msec 2023-07-22T12:28:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-07-22T12:28:52,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028932274 (current time:1690028932274). 2023-07-22T12:28:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:28:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690028932274 VERSION not specified, setting to 2 2023-07-22T12:28:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49271a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:52,284 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47136, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:28:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-07-22T12:28:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:28:52,289 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:28:52,290 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:28:52,292 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:28:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:28:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:28:52,701 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:28:52,703 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:28:52,703 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:28:52,704 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:52,704 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(327): No files under family: meta 2023-07-22T12:28:52,704 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:52,706 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/ded5dbf29e1c4c659ed9139c21ad4ef9] hfiles 2023-07-22T12:28:52,707 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:52,707 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/ded5dbf29e1c4c659ed9139c21ad4ef9 2023-07-22T12:28:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:28:53,117 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:28:53,119 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:28:53,120 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:28:53,120 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690028932274 2023-07-22T12:28:53,121 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028932274 2023-07-22T12:28:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:28:53,539 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:28:53,547 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:28:53,548 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028932274 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690028932274 2023-07-22T12:28:54,378 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:28:54,378 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-07-22T12:28:54,379 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028932274 table=backup:system type=FLUSH ttl=0 } in 2.0920 sec 2023-07-22T12:28:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-07-22T12:28:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028926894 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:28:54,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:28:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:28:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:28:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:28:54,841 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-07-22T12:28:54,841 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-07-22T12:28:54,843 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:54,849 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028926894 type: FLUSH version: 2 ttl: 0 2023-07-22T12:28:54,850 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system 2023-07-22T12:28:54,853 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:28:54,853 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:54,863 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:54,873 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/ded5dbf29e1c4c659ed9139c21ad4ef9 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/ded5dbf29e1c4c659ed9139c21ad4ef9 2023-07-22T12:28:54,874 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:54,874 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028926894 type: FLUSH version: 2 ttl: 0 2023-07-22T12:28:54,876 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690028934875"}]},"ts":"9223372036854775807"} 2023-07-22T12:28:54,927 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:28:54,927 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:28:54,928 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028934876"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028934876"}]},"ts":"1690028934876"} 2023-07-22T12:28:54,930 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:28:54,930 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:28:54,930 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:28:54,931 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:28:54,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028926894 type: FLUSH version: 2 ttl: 0 ) in 521 msec 2023-07-22T12:28:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-07-22T12:28:55,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028932274" 2023-07-22T12:28:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690028932274 2023-07-22T12:28:55,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:28:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=39, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:28:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:28:55,060 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:28:55,060 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028935060"}]},"ts":"1690028935060"} 2023-07-22T12:28:55,061 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:28:55,065 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:28:55,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN}] 2023-07-22T12:28:55,067 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN 2023-07-22T12:28:55,067 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:28:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:28:55,219 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:55,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:28:55,399 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:55,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:28:55,399 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:28:55,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:28:55,400 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,400 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,401 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,402 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName meta 2023-07-22T12:28:55,402 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:55,403 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:55,403 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,404 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName session 2023-07-22T12:28:55,404 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:55,412 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:55,412 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:28:55,413 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,415 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,416 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:28:55,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:55,419 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened ccd715104b6c80cc2e57c916da0834fd; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10101760480, jitterRate=-0.05920024216175079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:28:55,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:55,420 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., pid=41, masterSystemTime=1690028935373 2023-07-22T12:28:55,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:55,422 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:55,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPEN, openSeqNum=13, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:55,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=41, resume processing ppid=40 2023-07-22T12:28:55,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 204 msec 2023-07-22T12:28:55,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-07-22T12:28:55,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN in 362 msec 2023-07-22T12:28:55,430 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028935430"}]},"ts":"1690028935430"} 2023-07-22T12:28:55,431 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:28:55,433 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:28:55,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, state=SUCCESS; EnableTableProcedure table=backup:system in 383 msec 2023-07-22T12:28:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-07-22T12:28:55,664 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:28:55,664 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:28:55,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:28:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:28:55,673 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:28:55,673 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-07-22T12:28:55,675 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1690028926808 2023-07-22T12:28:55,689 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:55,689 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:55,707 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupRepair(57): Running stage 1 2023-07-22T12:28:55,724 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787c1d7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:55,725 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:28:55,727 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59612, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:28:55,731 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:55,733 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:55,774 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:28:55,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028935778 (current time:1690028935778). 2023-07-22T12:28:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:28:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:28:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4489a698, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:55,787 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:28:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-07-22T12:28:55,791 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:28:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:28:55,792 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:28:55,794 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:28:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:28:55,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-07-22T12:28:55,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-07-22T12:28:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:28:56,205 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:28:56,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd}] 2023-07-22T12:28:56,207 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:56,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-07-22T12:28:56,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:56,360 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-07-22T12:28:56,381 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/2edc8cf2329541ff8250d131aa3bb79f is 284, key is session:backup_1690028926808/session:context/1690028935674/Put/seqid=0 2023-07-22T12:28:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:28:56,792 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:28:56,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:28:56,800 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/2edc8cf2329541ff8250d131aa3bb79f as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:28:56,806 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f, entries=2, sequenceid=18, filesize=5.2 K 2023-07-22T12:28:56,807 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 447ms, sequenceid=18, compaction requested=false 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. for snapshot_backup_system completed. 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f] hfiles 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 for snapshot=snapshot_backup_system 2023-07-22T12:28:56,807 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f for snapshot=snapshot_backup_system 2023-07-22T12:28:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:28:57,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:57,220 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-07-22T12:28:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster(3994): Remote procedure done, pid=43 2023-07-22T12:28:57,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:57,221 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:57,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=43, resume processing ppid=42 2023-07-22T12:28:57,224 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:28:57,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd in 1.0170 sec 2023-07-22T12:28:57,225 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:28:57,226 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:28:57,226 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:28:57,227 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:28:57,285 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:28:57,645 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:28:57,653 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:28:57,653 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:28:57,674 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:28:57,674 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-07-22T12:28:57,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8860 sec 2023-07-22T12:28:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-07-22T12:28:57,897 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(115): Backup backup_1690028935759 started at 1690028937897. 2023-07-22T12:28:57,899 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(119): Backup session backup_1690028935759 has been started. Backup session finished. Status: FAILURE 2023-07-22T12:28:57,903 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:57,903 ERROR [Listener at jenkins-hbase3.apache.org/36493 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] ... 32 more 2023-07-22T12:28:57,910 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6332542f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:57,915 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776c9aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:57,916 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:28:57,918 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:59628, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:28:57,922 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:57,924 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47150, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1690028935759,startts=1690028937897,failedts=1690028937942,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1690028935759 2023-07-22T12:28:57,942 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:28:57,943 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:28:57,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:28:57,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:28:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:28:57,953 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028937953"}]},"ts":"1690028937953"} 2023-07-22T12:28:57,955 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:28:57,957 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:28:57,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN}] 2023-07-22T12:28:57,960 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN 2023-07-22T12:28:57,961 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:28:57,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:28:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:28:58,115 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:58,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing ccd715104b6c80cc2e57c916da0834fd, disabling compactions & flushes 2023-07-22T12:28:58,115 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:58,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:58,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. after waiting 0 ms 2023-07-22T12:28:58,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:58,115 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=289 B heapSize=952 B 2023-07-22T12:28:58,130 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/2036a814595a4095a476849dc91ee0eb is 79, key is startcode:hdfs://localhost:42225/backupUT/meta:startcode/1690028937901/Put/seqid=0 2023-07-22T12:28:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:28:58,536 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=75 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/2036a814595a4095a476849dc91ee0eb 2023-07-22T12:28:58,542 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2036a814595a4095a476849dc91ee0eb 2023-07-22T12:28:58,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/9ad83b873e4e4216b1639cceda557644 is 218, key is session:backup_1690028935759/session:context/1690028937898/Put/seqid=0 2023-07-22T12:28:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:28:58,952 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/9ad83b873e4e4216b1639cceda557644 2023-07-22T12:28:58,960 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9ad83b873e4e4216b1639cceda557644 2023-07-22T12:28:58,961 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/2036a814595a4095a476849dc91ee0eb as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/2036a814595a4095a476849dc91ee0eb 2023-07-22T12:28:58,970 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/2036a814595a4095a476849dc91ee0eb, entries=1, sequenceid=23, filesize=5.1 K 2023-07-22T12:28:58,971 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/9ad83b873e4e4216b1639cceda557644 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/9ad83b873e4e4216b1639cceda557644 2023-07-22T12:28:58,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/9ad83b873e4e4216b1639cceda557644, entries=1, sequenceid=23, filesize=5.2 K 2023-07-22T12:28:58,980 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(3022): Finished flush of dataSize ~289 B/289, heapSize ~920 B/920, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 865ms, sequenceid=23, compaction requested=true 2023-07-22T12:28:58,982 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2036a814595a4095a476849dc91ee0eb 2023-07-22T12:28:58,984 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:58,985 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:28:58,987 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9ad83b873e4e4216b1639cceda557644 2023-07-22T12:28:58,992 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=12 2023-07-22T12:28:58,992 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:28:58,993 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:28:58,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:28:58,995 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:28:58,996 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSED 2023-07-22T12:28:58,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=46, resume processing ppid=45 2023-07-22T12:28:58,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 1.0340 sec 2023-07-22T12:28:59,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-07-22T12:28:59,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN in 1.0410 sec 2023-07-22T12:28:59,001 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028939001"}]},"ts":"1690028939001"} 2023-07-22T12:28:59,003 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:28:59,004 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:28:59,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0570 sec 2023-07-22T12:28:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-07-22T12:28:59,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028939067 (current time:1690028939067). 2023-07-22T12:28:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:28:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690028939066 VERSION not specified, setting to 2 2023-07-22T12:28:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db32127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:28:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:28:59,075 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:28:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:28:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:28:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:28:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-07-22T12:28:59,079 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:28:59,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:28:59,080 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:28:59,082 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:28:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:28:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:28:59,492 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:28:59,493 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:28:59,493 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:28:59,494 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:59,495 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/2036a814595a4095a476849dc91ee0eb] hfiles 2023-07-22T12:28:59,495 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/2036a814595a4095a476849dc91ee0eb 2023-07-22T12:28:59,495 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:28:59,497 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/9ad83b873e4e4216b1639cceda557644] hfiles 2023-07-22T12:28:59,497 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:28:59,497 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:28:59,497 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/9ad83b873e4e4216b1639cceda557644 2023-07-22T12:28:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:28:59,904 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:28:59,905 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:28:59,906 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:28:59,906 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690028939066 2023-07-22T12:28:59,906 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028939066 2023-07-22T12:29:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:29:00,318 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:00,327 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:00,327 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028939066 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690028939066 2023-07-22T12:29:01,151 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:01,151 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-07-22T12:29:01,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028939066 table=backup:system type=FLUSH ttl=0 } in 2.0750 sec 2023-07-22T12:29:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-07-22T12:29:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=48, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028935778 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:29:01,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:29:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:29:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:29:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:29:01,611 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-07-22T12:29:01,611 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-07-22T12:29:01,613 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:01,616 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028935778 type: FLUSH version: 2 ttl: 0 2023-07-22T12:29:01,616 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system 2023-07-22T12:29:01,619 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:29:01,619 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:01,627 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/2036a814595a4095a476849dc91ee0eb to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/2036a814595a4095a476849dc91ee0eb 2023-07-22T12:29:01,627 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:01,631 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/9ad83b873e4e4216b1639cceda557644 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/9ad83b873e4e4216b1639cceda557644 2023-07-22T12:29:01,631 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:01,632 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028935778 type: FLUSH version: 2 ttl: 0 2023-07-22T12:29:01,633 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690028941633"}]},"ts":"9223372036854775807"} 2023-07-22T12:29:01,635 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:29:01,635 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:29:01,636 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028941634"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028941634"}]},"ts":"1690028941634"} 2023-07-22T12:29:01,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:29:01,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:29:01,637 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:29:01,637 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:29:01,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028935778 type: FLUSH version: 2 ttl: 0 ) in 445 msec 2023-07-22T12:29:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-07-22T12:29:01,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028939066" 2023-07-22T12:29:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690028939066 2023-07-22T12:29:01,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:29:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=49, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:29:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:29:01,810 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:29:01,810 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028941810"}]},"ts":"1690028941810"} 2023-07-22T12:29:01,812 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:29:01,814 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:29:01,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN}] 2023-07-22T12:29:01,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN 2023-07-22T12:29:01,816 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:29:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:29:01,968 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:01,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:29:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:29:02,126 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(130): Open backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:02,126 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7254): Opening region: {ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:29:02,127 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:29:02,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(888): Instantiated backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:29:02,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7296): checking encryption for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7299): checking classloading for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,129 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,130 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName meta 2023-07-22T12:29:02,130 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:02,131 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:29:02,131 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,132 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName session 2023-07-22T12:29:02,132 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:02,140 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:02,145 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:02,145 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:29:02,146 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,147 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:29:02,150 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1076): writing seq id for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:02,151 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): Opened ccd715104b6c80cc2e57c916da0834fd; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10336959360, jitterRate=-0.037295639514923096}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:29:02,152 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(992): Region open journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:02,153 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., pid=51, masterSystemTime=1690028942122 2023-07-22T12:29:02,155 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:02,155 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(158): Opened backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:02,155 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPEN, openSeqNum=27, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:02,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-07-22T12:29:02,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 187 msec 2023-07-22T12:29:02,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=50, resume processing ppid=49 2023-07-22T12:29:02,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, ppid=49, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN in 345 msec 2023-07-22T12:29:02,162 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028942161"}]},"ts":"1690028942161"} 2023-07-22T12:29:02,163 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:29:02,165 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:29:02,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, state=SUCCESS; EnableTableProcedure table=backup:system in 359 msec 2023-07-22T12:29:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-07-22T12:29:02,413 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:29:02,413 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:29:02,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:29:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:29:02,424 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:29:02,424 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-07-22T12:29:02,426 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1690028935759 2023-07-22T12:29:02,432 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:02,432 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:02,466 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupRepair(57): Running stage 2 2023-07-22T12:29:02,485 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7248c32f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:02,487 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:29:02,488 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56082, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:29:02,492 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:02,493 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:02,538 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:29:02,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028942542 (current time:1690028942542). 2023-07-22T12:29:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:29:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:29:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674f36ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:02,551 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:29:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-07-22T12:29:02,555 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:29:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:02,556 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:29:02,557 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:29:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:02,711 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-07-22T12:29:02,714 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56088, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-07-22T12:29:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:02,964 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:29:02,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd}] 2023-07-22T12:29:02,966 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:03,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-07-22T12:29:03,118 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:03,119 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-07-22T12:29:03,133 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/6770268a1cc643098aac099ee6e573b4 is 284, key is session:backup_1690028935759/session:context/1690028942425/Put/seqid=0 2023-07-22T12:29:03,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:03,538 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:03,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:03,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/6770268a1cc643098aac099ee6e573b4 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:03,553 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4, entries=2, sequenceid=32, filesize=5.2 K 2023-07-22T12:29:03,554 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 435ms, sequenceid=32, compaction requested=true 2023-07-22T12:29:03,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:29:03,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:03,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. for snapshot_backup_system completed. 2023-07-22T12:29:03,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:29:03,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:29:03,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:29:03,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4] hfiles 2023-07-22T12:29:03,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 for snapshot=snapshot_backup_system 2023-07-22T12:29:03,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f for snapshot=snapshot_backup_system 2023-07-22T12:29:03,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4 for snapshot=snapshot_backup_system 2023-07-22T12:29:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:03,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:03,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-07-22T12:29:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster(3994): Remote procedure done, pid=53 2023-07-22T12:29:03,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:03,966 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:03,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-07-22T12:29:03,969 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:29:03,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd in 1.0030 sec 2023-07-22T12:29:03,970 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:29:03,970 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:29:03,970 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:29:03,971 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:29:04,381 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:04,389 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:04,389 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:29:04,496 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:29:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:05,209 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:05,209 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-07-22T12:29:05,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6570 sec 2023-07-22T12:29:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-07-22T12:29:06,662 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(115): Backup backup_1690028942521 started at 1690028946662. 2023-07-22T12:29:06,664 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(119): Backup session backup_1690028942521 has been started. 2023-07-22T12:29:06,666 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-07-22T12:29:06,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:29:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:29:06,685 INFO [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:29:06,685 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:29:06,685 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:29:06,685 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:29:06,686 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:29:06,689 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,689 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:29:06,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:29:06,689 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:29:06,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:29:06,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:29:06,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:29:06,690 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:29:06,692 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:29:06,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:29:06,693 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:29:06,693 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:29:06,693 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:29:06,693 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:29:06,693 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36719,1690028908802' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:29:06,696 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:06,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:29:06,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:29:06,696 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:29:06,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:29:06,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:29:06,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:06,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,698 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:29:06,698 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36719,1690028908802' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:29:06,698 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4e08b88[Count = 0] remaining members to acquire global barrier 2023-07-22T12:29:06,698 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:29:06,698 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:06,700 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:06,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:06,700 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:29:06,700 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:29:06,701 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:29:06,701 DEBUG [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,702 INFO [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690028911218 highest: 1690028911218 on jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,702 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta:.meta(num 1690028911793) roll requested 2023-07-22T12:29:06,713 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028946702.meta, exclude list is [], retry=0 2023-07-22T12:29:06,717 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:29:06,720 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028911793.meta with entries=56, filesize=15.96 KB; new WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028946702.meta 2023-07-22T12:29:06,721 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:29:06,721 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028911793.meta is not closed yet, will try archiving it next time 2023-07-22T12:29:06,722 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36719%2C1690028908802:(num 1690028911218) roll requested 2023-07-22T12:29:06,735 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028946722, exclude list is [], retry=0 2023-07-22T12:29:06,738 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:29:06,741 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028911218 with entries=40, filesize=9.34 KB; new WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028946722 2023-07-22T12:29:06,741 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:29:06,741 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028911218 is not closed yet, will try archiving it next time 2023-07-22T12:29:06,742 DEBUG [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:29:06,742 INFO [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690028946722 on jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,744 DEBUG [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:29:06,746 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56094, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-07-22T12:29:06,762 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:29:06,762 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36719,1690028908802' in zk 2023-07-22T12:29:06,764 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:29:06,764 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:29:06,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,765 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:29:06,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:29:06,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:29:06,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:29:06,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:29:06,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:06,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:29:06,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:06,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,768 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36719,1690028908802': 2023-07-22T12:29:06,768 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36719,1690028908802' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:29:06,768 INFO [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:29:06,768 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:29:06,768 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:29:06,768 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:29:06,768 INFO [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:29:06,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:29:06,772 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:29:06,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:29:06,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:29:06,772 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:29:06,773 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:29:06,773 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:29:06,773 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:29:06,773 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:29:06,773 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:06,774 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:29:06,774 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:06,774 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:29:06,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:06,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,784 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:29:06,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:29:06,784 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:29:06,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:29:06,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:29:06,785 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:29:06,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:29:06,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done Backup session finished. Status: FAILURE 2023-07-22T12:29:06,894 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:06,894 ERROR [Listener at jenkins-hbase3.apache.org/36493 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] ... 32 more 2023-07-22T12:29:06,901 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fff1992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:06,907 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@101cc89b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:06,908 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:29:06,910 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:29:06,914 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:06,915 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45148, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1690028942521,startts=1690028946662,failedts=1690028946932,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1690028942521 2023-07-22T12:29:06,932 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:29:06,933 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:29:06,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:29:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:29:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:29:06,943 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028946943"}]},"ts":"1690028946943"} 2023-07-22T12:29:06,945 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:29:06,954 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:29:06,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN}] 2023-07-22T12:29:06,956 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN 2023-07-22T12:29:06,956 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:06,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:29:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:29:07,110 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:07,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing ccd715104b6c80cc2e57c916da0834fd, disabling compactions & flushes 2023-07-22T12:29:07,110 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:07,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:07,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. after waiting 0 ms 2023-07-22T12:29:07,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:07,110 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-07-22T12:29:07,125 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/c0aca02db34e43e09af2a5539676857d is 116, key is rslogts:hdfs://localhost:42225/backupUT\x00jenkins-hbase3.apache.org:36719/meta:rs-log-ts/1690028946761/Put/seqid=0 2023-07-22T12:29:07,135 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/c0aca02db34e43e09af2a5539676857d 2023-07-22T12:29:07,141 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c0aca02db34e43e09af2a5539676857d 2023-07-22T12:29:07,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/20260c4d8da94bdfb6394278a4c16a12 is 218, key is session:backup_1690028942521/session:context/1690028946663/Put/seqid=0 2023-07-22T12:29:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:29:07,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:29:07,552 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/20260c4d8da94bdfb6394278a4c16a12 2023-07-22T12:29:07,559 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 20260c4d8da94bdfb6394278a4c16a12 2023-07-22T12:29:07,560 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/c0aca02db34e43e09af2a5539676857d as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/c0aca02db34e43e09af2a5539676857d 2023-07-22T12:29:07,566 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/c0aca02db34e43e09af2a5539676857d, entries=2, sequenceid=38, filesize=5.3 K 2023-07-22T12:29:07,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/20260c4d8da94bdfb6394278a4c16a12 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/20260c4d8da94bdfb6394278a4c16a12 2023-07-22T12:29:07,573 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/20260c4d8da94bdfb6394278a4c16a12, entries=1, sequenceid=38, filesize=5.2 K 2023-07-22T12:29:07,574 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 464ms, sequenceid=38, compaction requested=true 2023-07-22T12:29:07,576 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c0aca02db34e43e09af2a5539676857d 2023-07-22T12:29:07,578 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:07,579 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:07,580 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:07,582 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 20260c4d8da94bdfb6394278a4c16a12 2023-07-22T12:29:07,586 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/recovered.edits/41.seqid, newMaxSeqId=41, maxSeqId=26 2023-07-22T12:29:07,586 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:07,587 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:07,587 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:07,589 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:07,589 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSED 2023-07-22T12:29:07,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=56, resume processing ppid=55 2023-07-22T12:29:07,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 632 msec 2023-07-22T12:29:07,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-07-22T12:29:07,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN in 638 msec 2023-07-22T12:29:07,594 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028947593"}]},"ts":"1690028947593"} 2023-07-22T12:29:07,595 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:29:07,598 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:29:07,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; DisableTableProcedure table=backup:system in 659 msec 2023-07-22T12:29:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-07-22T12:29:08,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028948057 (current time:1690028948057). 2023-07-22T12:29:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:29:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690028948056 VERSION not specified, setting to 2 2023-07-22T12:29:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2914eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:08,069 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:45164, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:08,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:29:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-07-22T12:29:08,073 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:29:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:29:08,074 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:29:08,076 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:29:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:29:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:29:08,485 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:29:08,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:29:08,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:29:08,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:08,487 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/c0aca02db34e43e09af2a5539676857d] hfiles 2023-07-22T12:29:08,487 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/c0aca02db34e43e09af2a5539676857d 2023-07-22T12:29:08,487 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:08,489 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/20260c4d8da94bdfb6394278a4c16a12, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886] hfiles 2023-07-22T12:29:08,489 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/20260c4d8da94bdfb6394278a4c16a12 2023-07-22T12:29:08,489 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:08,489 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:08,489 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:29:08,896 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:29:08,897 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:29:08,897 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:29:08,897 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690028948056 2023-07-22T12:29:08,898 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028948056 2023-07-22T12:29:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:29:09,309 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:09,317 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:09,317 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028948056 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690028948056 2023-07-22T12:29:10,136 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:10,137 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-07-22T12:29:10,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028948056 table=backup:system type=FLUSH ttl=0 } in 2.0670 sec 2023-07-22T12:29:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-07-22T12:29:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=58, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028942542 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:29:10,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:29:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:29:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:29:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:29:10,601 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-07-22T12:29:10,602 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-07-22T12:29:10,603 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:10,605 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028942542 type: FLUSH version: 2 ttl: 0 2023-07-22T12:29:10,606 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system 2023-07-22T12:29:10,608 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:29:10,608 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:10,616 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/c0aca02db34e43e09af2a5539676857d to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/c0aca02db34e43e09af2a5539676857d 2023-07-22T12:29:10,616 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:10,620 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/20260c4d8da94bdfb6394278a4c16a12 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/20260c4d8da94bdfb6394278a4c16a12 2023-07-22T12:29:10,620 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:10,621 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028942542 type: FLUSH version: 2 ttl: 0 2023-07-22T12:29:10,622 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690028950622"}]},"ts":"9223372036854775807"} 2023-07-22T12:29:10,624 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:29:10,624 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:29:10,625 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028950623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028950623"}]},"ts":"1690028950623"} 2023-07-22T12:29:10,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:29:10,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:29:10,626 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:29:10,626 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:29:10,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028942542 type: FLUSH version: 2 ttl: 0 ) in 440 msec 2023-07-22T12:29:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-07-22T12:29:10,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028948056" 2023-07-22T12:29:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690028948056 2023-07-22T12:29:10,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:29:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=59, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:29:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:29:10,802 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:29:10,802 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028950802"}]},"ts":"1690028950802"} 2023-07-22T12:29:10,804 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:29:10,806 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:29:10,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN}] 2023-07-22T12:29:10,808 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN 2023-07-22T12:29:10,809 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:29:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:29:10,960 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:10,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:29:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:29:11,117 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(130): Open backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:11,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7254): Opening region: {ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:29:11,118 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:29:11,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(888): Instantiated backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:29:11,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7296): checking encryption for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7299): checking classloading for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,119 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,121 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName meta 2023-07-22T12:29:11,121 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:11,121 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:29:11,122 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,122 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName session 2023-07-22T12:29:11,122 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:11,131 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:11,135 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:11,141 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:11,141 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:29:11,146 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,148 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:29:11,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1076): writing seq id for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:11,152 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): Opened ccd715104b6c80cc2e57c916da0834fd; next sequenceid=42; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9829017440, jitterRate=-0.08460141718387604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:29:11,153 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(992): Region open journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:11,154 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., pid=61, masterSystemTime=1690028951114 2023-07-22T12:29:11,158 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(403): Add compact mark for store ccd715104b6c80cc2e57c916da0834fd:session, priority=-2147483648, current under compaction store size is 1 2023-07-22T12:29:11,159 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-07-22T12:29:11,160 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-07-22T12:29:11,161 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:11,162 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(158): Opened backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:11,162 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPEN, openSeqNum=42, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:11,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=61, resume processing ppid=60 2023-07-22T12:29:11,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, ppid=60, state=SUCCESS; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 202 msec 2023-07-22T12:29:11,166 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 15770 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-07-22T12:29:11,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=60, resume processing ppid=59 2023-07-22T12:29:11,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, ppid=59, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN in 360 msec 2023-07-22T12:29:11,169 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028951169"}]},"ts":"1690028951169"} 2023-07-22T12:29:11,169 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.HStore(1534): ccd715104b6c80cc2e57c916da0834fd/session is initiating minor compaction (all files) 2023-07-22T12:29:11,169 INFO [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of ccd715104b6c80cc2e57c916da0834fd/session in backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:11,170 INFO [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4] into tmpdir=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp, totalSize=15.4 K 2023-07-22T12:29:11,171 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:29:11,171 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8973d311c3014795b5c35b33ce8fd886, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1690028926876 2023-07-22T12:29:11,172 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2edc8cf2329541ff8250d131aa3bb79f, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1690028935674 2023-07-22T12:29:11,172 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6770268a1cc643098aac099ee6e573b4, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1690028942425 2023-07-22T12:29:11,172 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:29:11,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, state=SUCCESS; EnableTableProcedure table=backup:system in 375 msec 2023-07-22T12:29:11,200 INFO [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd715104b6c80cc2e57c916da0834fd#session#compaction#8 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-07-22T12:29:11,201 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/0bafe9ca03c0427f87758e00587e8db0 is 284, key is session:backup_1690028926808/session:context/1690028935674/Put/seqid=0 2023-07-22T12:29:11,212 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0bafe9ca03c0427f87758e00587e8db0 2023-07-22T12:29:11,213 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/0bafe9ca03c0427f87758e00587e8db0 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/0bafe9ca03c0427f87758e00587e8db0 2023-07-22T12:29:11,233 INFO [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in ccd715104b6c80cc2e57c916da0834fd/session of ccd715104b6c80cc2e57c916da0834fd into 0bafe9ca03c0427f87758e00587e8db0(size=5.6 K), total size for store is 5.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-07-22T12:29:11,233 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:11,234 INFO [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., storeName=ccd715104b6c80cc2e57c916da0834fd/session, priority=13, startTime=1690028951157; duration=0sec 2023-07-22T12:29:11,235 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-07-22T12:29:11,235 DEBUG [RS:0;jenkins-hbase3:36719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd715104b6c80cc2e57c916da0834fd:session 2023-07-22T12:29:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-07-22T12:29:11,405 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:29:11,405 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:29:11,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:29:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:29:11,413 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:29:11,413 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-07-22T12:29:11,415 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1690028942521 2023-07-22T12:29:11,420 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:11,421 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:11,465 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupRepair(57): Running stage 3 2023-07-22T12:29:11,485 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68935a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:11,486 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:29:11,488 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:29:11,491 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:11,493 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33782, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:11,539 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-07-22T12:29:11,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028951542 (current time:1690028951542). 2023-07-22T12:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:29:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-07-22T12:29:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c99da82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:11,557 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33788, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:29:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-07-22T12:29:11,561 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:29:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:11,562 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:29:11,564 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:29:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:11,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:11,979 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:29:11,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd}] 2023-07-22T12:29:11,982 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:12,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-07-22T12:29:12,133 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:12,134 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-07-22T12:29:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/f51b983ec13445eda6d7939681e1bf0a is 284, key is session:backup_1690028942521/session:context/1690028951414/Put/seqid=0 2023-07-22T12:29:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:12,544 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:12,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:12,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/f51b983ec13445eda6d7939681e1bf0a as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:12,559 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a, entries=2, sequenceid=48, filesize=5.2 K 2023-07-22T12:29:12,560 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 426ms, sequenceid=48, compaction requested=false 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. for snapshot_backup_system completed. 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.' region-info for snapshot=snapshot_backup_system 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/0bafe9ca03c0427f87758e00587e8db0, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a] hfiles 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/0bafe9ca03c0427f87758e00587e8db0 for snapshot=snapshot_backup_system 2023-07-22T12:29:12,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a for snapshot=snapshot_backup_system 2023-07-22T12:29:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:12,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:12,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-07-22T12:29:12,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster(3994): Remote procedure done, pid=63 2023-07-22T12:29:12,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:12,968 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:12,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-07-22T12:29:12,971 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:29:12,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure ccd715104b6c80cc2e57c916da0834fd in 989 msec 2023-07-22T12:29:12,971 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:29:12,972 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:29:12,972 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-07-22T12:29:12,973 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system 2023-07-22T12:29:12,987 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:12,993 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:12,994 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/snapshot_backup_system 2023-07-22T12:29:13,090 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:29:13,648 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-07-22T12:29:13,648 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-07-22T12:29:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:13,816 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:13,816 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-07-22T12:29:13,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2570 sec 2023-07-22T12:29:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-07-22T12:29:15,672 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(115): Backup backup_1690028951525 started at 1690028955672. 2023-07-22T12:29:15,674 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(119): Backup session backup_1690028951525 has been started. 2023-07-22T12:29:15,677 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-07-22T12:29:15,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-07-22T12:29:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-07-22T12:29:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-07-22T12:29:15,678 INFO [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-07-22T12:29:15,678 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-07-22T12:29:15,679 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-07-22T12:29:15,679 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-07-22T12:29:15,680 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:29:15,681 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,681 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:29:15,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:29:15,682 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-07-22T12:29:15,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-07-22T12:29:15,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-07-22T12:29:15,682 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-07-22T12:29:15,683 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-07-22T12:29:15,683 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-07-22T12:29:15,683 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-07-22T12:29:15,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-07-22T12:29:15,683 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-07-22T12:29:15,683 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-07-22T12:29:15,684 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-07-22T12:29:15,684 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-07-22T12:29:15,684 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36719,1690028908802' joining acquired barrier for procedure (rolllog) in zk 2023-07-22T12:29:15,688 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:15,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:29:15,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:29:15,688 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-07-22T12:29:15,688 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:29:15,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:29:15,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:15,689 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:29:15,690 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36719,1690028908802' joining acquired barrier for procedure 'rolllog' on coordinator 2023-07-22T12:29:15,690 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4a3f6371[Count = 0] remaining members to acquire global barrier 2023-07-22T12:29:15,690 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-07-22T12:29:15,690 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:15,691 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:15,691 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-07-22T12:29:15,692 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-07-22T12:29:15,692 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-07-22T12:29:15,692 DEBUG [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,692 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-07-22T12:29:15,692 INFO [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1690028946722 highest: 1690028946722 on jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,692 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta:.meta(num 1690028946702) roll requested 2023-07-22T12:29:15,703 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028955692.meta, exclude list is [], retry=0 2023-07-22T12:29:15,708 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:29:15,710 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028946702.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028955692.meta 2023-07-22T12:29:15,710 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:29:15,710 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028946702.meta is not closed yet, will try archiving it next time 2023-07-22T12:29:15,710 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36719%2C1690028908802:(num 1690028946722) roll requested 2023-07-22T12:29:15,712 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta.1690028946702.meta not finished, retry = 0 2023-07-22T12:29:15,722 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028955710, exclude list is [], retry=0 2023-07-22T12:29:15,726 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK] 2023-07-22T12:29:15,728 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028946722 with entries=13, filesize=3.42 KB; new WAL /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028955710 2023-07-22T12:29:15,728 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:40609,DS-36a60c2b-4761-4ba0-8835-86cd9dceafa5,DISK]] 2023-07-22T12:29:15,728 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/WALs/jenkins-hbase3.apache.org,36719,1690028908802/jenkins-hbase3.apache.org%2C36719%2C1690028908802.1690028946722 is not closed yet, will try archiving it next time 2023-07-22T12:29:15,732 DEBUG [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-07-22T12:29:15,732 INFO [rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1690028955710 on jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,741 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-07-22T12:29:15,741 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36719,1690028908802' in zk 2023-07-22T12:29:15,744 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-07-22T12:29:15,744 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:29:15,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,744 DEBUG [member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-07-22T12:29:15,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:29:15,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:29:15,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:29:15,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:29:15,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:15,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:29:15,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:15,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36719,1690028908802': 2023-07-22T12:29:15,747 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36719,1690028908802' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-07-22T12:29:15,747 INFO [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-07-22T12:29:15,747 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-07-22T12:29:15,747 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-07-22T12:29:15,747 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-07-22T12:29:15,747 INFO [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-07-22T12:29:15,749 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-07-22T12:29:15,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:29:15,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-07-22T12:29:15,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-07-22T12:29:15,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-07-22T12:29:15,749 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:29:15,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:29:15,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-07-22T12:29:15,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-07-22T12:29:15,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:15,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-07-22T12:29:15,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:15,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-07-22T12:29:15,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-07-22T12:29:15,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:15,757 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-07-22T12:29:15,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-07-22T12:29:15,757 DEBUG [(jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-07-22T12:29:15,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-07-22T12:29:15,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-07-22T12:29:15,758 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-07-22T12:29:15,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-07-22T12:29:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-07-22T12:29:15,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } 2023-07-22T12:29:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028955862 (current time:1690028955862). 2023-07-22T12:29:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:29:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690028955861_default_test-1690028918084 VERSION not specified, setting to 2 2023-07-22T12:29:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154a11fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:15,870 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33792, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:29:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } 2023-07-22T12:29:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:29:15,875 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:29:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:15,876 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:29:15,877 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:29:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:16,284 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:29:16,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure d465cc75c102ecbb2abf7437d3ee9ec0}] 2023-07-22T12:29:16,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:29:16,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-07-22T12:29:16,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:16,439 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing d465cc75c102ecbb2abf7437d3ee9ec0 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:29:16,455 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/.tmp/f/b4f8cdd8a5dc4c0d84fc2c28f1304547 is 37, key is row10/f:q1/1690028919237/Put/seqid=0 2023-07-22T12:29:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:16,860 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/.tmp/f/b4f8cdd8a5dc4c0d84fc2c28f1304547 2023-07-22T12:29:16,865 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b4f8cdd8a5dc4c0d84fc2c28f1304547 2023-07-22T12:29:16,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/.tmp/f/b4f8cdd8a5dc4c0d84fc2c28f1304547 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/f/b4f8cdd8a5dc4c0d84fc2c28f1304547 2023-07-22T12:29:16,871 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/f/b4f8cdd8a5dc4c0d84fc2c28f1304547, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:29:16,872 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for d465cc75c102ecbb2abf7437d3ee9ec0 in 433ms, sequenceid=103, compaction requested=false 2023-07-22T12:29:16,873 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for d465cc75c102ecbb2abf7437d3ee9ec0: 2023-07-22T12:29:16,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. for snapshot_1690028955861_default_test-1690028918084 completed. 2023-07-22T12:29:16,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.' region-info for snapshot=snapshot_1690028955861_default_test-1690028918084 2023-07-22T12:29:16,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:29:16,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/f/b4f8cdd8a5dc4c0d84fc2c28f1304547] hfiles 2023-07-22T12:29:16,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/f/b4f8cdd8a5dc4c0d84fc2c28f1304547 for snapshot=snapshot_1690028955861_default_test-1690028918084 2023-07-22T12:29:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:17,279 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:17,279 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-07-22T12:29:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster(3994): Remote procedure done, pid=65 2023-07-22T12:29:17,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690028955861_default_test-1690028918084 on region d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:29:17,281 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:29:17,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=65, resume processing ppid=64 2023-07-22T12:29:17,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure d465cc75c102ecbb2abf7437d3ee9ec0 in 998 msec 2023-07-22T12:29:17,284 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:29:17,284 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:29:17,285 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:29:17,285 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690028955861_default_test-1690028918084 2023-07-22T12:29:17,286 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_1690028955861_default_test-1690028918084 2023-07-22T12:29:17,696 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:17,702 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:17,702 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_1690028955861_default_test-1690028918084 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/snapshot_1690028955861_default_test-1690028918084 2023-07-22T12:29:17,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:18,520 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:18,520 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-07-22T12:29:18,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1690028955861_default_test-1690028918084 table=test-1690028918084 type=FLUSH ttl=0 } in 2.6490 sec 2023-07-22T12:29:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-07-22T12:29:19,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } 2023-07-22T12:29:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028959983 (current time:1690028959983). 2023-07-22T12:29:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:29:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1690028959982_ns2_test-16900289180841 VERSION not specified, setting to 2 2023-07-22T12:29:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1ec12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:19,994 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33798, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:29:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } 2023-07-22T12:29:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-07-22T12:29:20,001 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:29:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:29:20,002 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:29:20,004 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:29:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:29:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:29:20,409 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-07-22T12:29:20,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 62c1d4e2655165c29f9a13c43a911f4c}] 2023-07-22T12:29:20,413 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:29:20,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36719 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-07-22T12:29:20,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:20,567 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing 62c1d4e2655165c29f9a13c43a911f4c 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-07-22T12:29:20,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/.tmp/f/167b6fcefa344acaa760c35269fcc9b9 is 37, key is row10/f:q1/1690028921452/Put/seqid=0 2023-07-22T12:29:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:29:20,991 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/.tmp/f/167b6fcefa344acaa760c35269fcc9b9 2023-07-22T12:29:20,997 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 167b6fcefa344acaa760c35269fcc9b9 2023-07-22T12:29:20,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/.tmp/f/167b6fcefa344acaa760c35269fcc9b9 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/f/167b6fcefa344acaa760c35269fcc9b9 2023-07-22T12:29:21,004 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/f/167b6fcefa344acaa760c35269fcc9b9, entries=99, sequenceid=103, filesize=8.2 K 2023-07-22T12:29:21,005 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 62c1d4e2655165c29f9a13c43a911f4c in 438ms, sequenceid=103, compaction requested=false 2023-07-22T12:29:21,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 62c1d4e2655165c29f9a13c43a911f4c: 2023-07-22T12:29:21,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. for snapshot_1690028959982_ns2_test-16900289180841 completed. 2023-07-22T12:29:21,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c.' region-info for snapshot=snapshot_1690028959982_ns2_test-16900289180841 2023-07-22T12:29:21,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-07-22T12:29:21,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/f/167b6fcefa344acaa760c35269fcc9b9] hfiles 2023-07-22T12:29:21,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/f/167b6fcefa344acaa760c35269fcc9b9 for snapshot=snapshot_1690028959982_ns2_test-16900289180841 2023-07-22T12:29:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:29:21,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:21,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-07-22T12:29:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster(3994): Remote procedure done, pid=67 2023-07-22T12:29:21,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1690028959982_ns2_test-16900289180841 on region 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:29:21,417 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:29:21,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=67, resume processing ppid=66 2023-07-22T12:29:21,420 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-07-22T12:29:21,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure 62c1d4e2655165c29f9a13c43a911f4c in 1.0090 sec 2023-07-22T12:29:21,421 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:29:21,421 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:29:21,421 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1690028959982_ns2_test-16900289180841 2023-07-22T12:29:21,422 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_1690028959982_ns2_test-16900289180841 2023-07-22T12:29:21,833 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:21,839 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:21,840 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/snapshot_1690028959982_ns2_test-16900289180841 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/snapshot_1690028959982_ns2_test-16900289180841 2023-07-22T12:29:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-07-22T12:29:22,660 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:22,661 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-07-22T12:29:22,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1690028959982_ns2_test-16900289180841 table=ns2:test-16900289180841 type=FLUSH ttl=0 } in 2.6650 sec 2023-07-22T12:29:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 Backup session finished. Status: FAILURE 2023-07-22T12:29:24,109 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:24,109 ERROR [Listener at jenkins-hbase3.apache.org/36493 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] ... 32 more 2023-07-22T12:29:24,117 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549d707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:24,122 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b24a6f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:24,123 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=MasterService, sasl=false 2023-07-22T12:29:24,124 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56870, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-07-22T12:29:24,128 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:24,130 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42008, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1690028951525,startts=1690028955672,failedts=1690028964146,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1690028951525 2023-07-22T12:29:24,146 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-07-22T12:29:24,147 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-07-22T12:29:24,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-07-22T12:29:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-07-22T12:29:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:29:24,161 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028964161"}]},"ts":"1690028964161"} 2023-07-22T12:29:24,162 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-07-22T12:29:24,165 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-07-22T12:29:24,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN}] 2023-07-22T12:29:24,167 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN 2023-07-22T12:29:24,168 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:24,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:29:24,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:29:24,321 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:24,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing ccd715104b6c80cc2e57c916da0834fd, disabling compactions & flushes 2023-07-22T12:29:24,321 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:24,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:24,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. after waiting 0 ms 2023-07-22T12:29:24,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:24,321 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-07-22T12:29:24,336 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/6fc8113624e3474197ac8a1cdc2b0777 is 116, key is rslogts:hdfs://localhost:42225/backupUT\x00jenkins-hbase3.apache.org:36719/meta:rs-log-ts/1690028955740/Put/seqid=0 2023-07-22T12:29:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:29:24,741 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/6fc8113624e3474197ac8a1cdc2b0777 2023-07-22T12:29:24,748 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6fc8113624e3474197ac8a1cdc2b0777 2023-07-22T12:29:24,751 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/8b700b5dc4444c349569dc3b3288d291 is 218, key is session:backup_1690028951525/session:context/1690028955673/Put/seqid=0 2023-07-22T12:29:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:29:25,156 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/8b700b5dc4444c349569dc3b3288d291 2023-07-22T12:29:25,162 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8b700b5dc4444c349569dc3b3288d291 2023-07-22T12:29:25,163 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/meta/6fc8113624e3474197ac8a1cdc2b0777 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/6fc8113624e3474197ac8a1cdc2b0777 2023-07-22T12:29:25,170 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/6fc8113624e3474197ac8a1cdc2b0777, entries=2, sequenceid=54, filesize=5.3 K 2023-07-22T12:29:25,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/8b700b5dc4444c349569dc3b3288d291 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8b700b5dc4444c349569dc3b3288d291 2023-07-22T12:29:25,176 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8b700b5dc4444c349569dc3b3288d291, entries=1, sequenceid=54, filesize=5.2 K 2023-07-22T12:29:25,177 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 855ms, sequenceid=54, compaction requested=true 2023-07-22T12:29:25,178 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6fc8113624e3474197ac8a1cdc2b0777 2023-07-22T12:29:25,180 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:25,181 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:25,183 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:25,184 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4] to archive 2023-07-22T12:29:25,185 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-07-22T12:29:25,186 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:25,189 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8973d311c3014795b5c35b33ce8fd886 2023-07-22T12:29:25,189 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:25,190 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/2edc8cf2329541ff8250d131aa3bb79f 2023-07-22T12:29:25,190 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:25,191 DEBUG [StoreCloser-backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/6770268a1cc643098aac099ee6e573b4 2023-07-22T12:29:25,219 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0bafe9ca03c0427f87758e00587e8db0 2023-07-22T12:29:25,221 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:25,222 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8b700b5dc4444c349569dc3b3288d291 2023-07-22T12:29:25,226 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/recovered.edits/57.seqid, newMaxSeqId=57, maxSeqId=41 2023-07-22T12:29:25,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:25,228 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:25,228 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:25,229 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:25,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=CLOSED 2023-07-22T12:29:25,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=70, resume processing ppid=69 2023-07-22T12:29:25,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 1.0620 sec 2023-07-22T12:29:25,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=69, resume processing ppid=68 2023-07-22T12:29:25,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, UNASSIGN in 1.0670 sec 2023-07-22T12:29:25,235 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028965235"}]},"ts":"1690028965235"} 2023-07-22T12:29:25,236 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-07-22T12:29:25,239 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-07-22T12:29:25,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=68, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0830 sec 2023-07-22T12:29:25,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-07-22T12:29:25,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1690028965282 (current time:1690028965282). 2023-07-22T12:29:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-07-22T12:29:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1690028965281 VERSION not specified, setting to 2 2023-07-22T12:29:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17eccc61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-07-22T12:29:25,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.RpcConnection(122): Using SIMPLE authentication for service=ClientService, sasl=false 2023-07-22T12:29:25,293 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:42018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-07-22T12:29:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-07-22T12:29:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=71, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-07-22T12:29:25,297 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-07-22T12:29:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:29:25,298 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-07-22T12:29:25,300 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-07-22T12:29:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:29:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:29:25,710 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-07-22T12:29:25,711 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-07-22T12:29:25,711 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-07-22T12:29:25,711 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:25,712 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/6fc8113624e3474197ac8a1cdc2b0777] hfiles 2023-07-22T12:29:25,712 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/6fc8113624e3474197ac8a1cdc2b0777 2023-07-22T12:29:25,712 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:25,714 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/0bafe9ca03c0427f87758e00587e8db0, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8b700b5dc4444c349569dc3b3288d291, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a] hfiles 2023-07-22T12:29:25,714 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/0bafe9ca03c0427f87758e00587e8db0 2023-07-22T12:29:25,714 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8b700b5dc4444c349569dc3b3288d291 2023-07-22T12:29:25,714 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:29:25,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-07-22T12:29:26,120 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-07-22T12:29:26,121 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-07-22T12:29:26,122 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-07-22T12:29:26,122 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1690028965281 2023-07-22T12:29:26,123 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028965281 2023-07-22T12:29:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:29:26,533 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-07-22T12:29:26,542 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-07-22T12:29:26,542 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1690028965281 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1690028965281 2023-07-22T12:29:26,965 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-07-22T12:29:26,965 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-07-22T12:29:26,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1690028965281 table=backup:system type=FLUSH ttl=0 } in 1.6700 sec 2023-07-22T12:29:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-07-22T12:29:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=72, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028951542 type: FLUSH version: 2 ttl: 0 ) 2023-07-22T12:29:27,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-07-22T12:29:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:29:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:29:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:29:27,826 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-07-22T12:29:27,826 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-07-22T12:29:27,828 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-07-22T12:29:27,830 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028951542 type: FLUSH version: 2 ttl: 0 2023-07-22T12:29:27,830 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system 2023-07-22T12:29:27,833 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-07-22T12:29:27,833 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:27,843 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/6fc8113624e3474197ac8a1cdc2b0777 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/meta/6fc8113624e3474197ac8a1cdc2b0777 2023-07-22T12:29:27,843 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:27,847 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8b700b5dc4444c349569dc3b3288d291 to hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/archive/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/8b700b5dc4444c349569dc3b3288d291 2023-07-22T12:29:27,847 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:27,847 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028951542 type: FLUSH version: 2 ttl: 0 2023-07-22T12:29:27,848 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1690028967848"}]},"ts":"9223372036854775807"} 2023-07-22T12:29:27,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-07-22T12:29:27,851 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:29:27,851 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1690028967849"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1690028967849"}]},"ts":"1690028967849"} 2023-07-22T12:29:27,852 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-07-22T12:29:27,852 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-07-22T12:29:27,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''}] 2023-07-22T12:29:27,852 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-07-22T12:29:27,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=72, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1690028951542 type: FLUSH version: 2 ttl: 0 ) in 442 msec 2023-07-22T12:29:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-07-22T12:29:28,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1690028965281" 2023-07-22T12:29:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1690028965281 2023-07-22T12:29:28,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-07-22T12:29:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] procedure2.ProcedureExecutor(1033): Stored pid=73, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-07-22T12:29:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-07-22T12:29:28,026 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-07-22T12:29:28,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028968026"}]},"ts":"1690028968026"} 2023-07-22T12:29:28,028 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-07-22T12:29:28,030 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-07-22T12:29:28,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN}] 2023-07-22T12:29:28,033 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN 2023-07-22T12:29:28,034 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-07-22T12:29:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-07-22T12:29:28,185 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:28,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802}] 2023-07-22T12:29:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-07-22T12:29:28,343 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(130): Open backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:28,343 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7254): Opening region: {ENCODED => ccd715104b6c80cc2e57c916da0834fd, NAME => 'backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.', STARTKEY => '', ENDKEY => ''} 2023-07-22T12:29:28,343 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-07-22T12:29:28,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(888): Instantiated backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-07-22T12:29:28,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7296): checking encryption for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7299): checking classloading for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,345 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,346 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName meta 2023-07-22T12:29:28,346 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:28,347 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:29:28,347 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,347 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd715104b6c80cc2e57c916da0834fd columnFamilyName session 2023-07-22T12:29:28,347 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-07-22T12:29:28,354 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/0bafe9ca03c0427f87758e00587e8db0 2023-07-22T12:29:28,359 DEBUG [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:28,359 INFO [StoreOpener-ccd715104b6c80cc2e57c916da0834fd-1 {}] regionserver.HStore(324): Store=ccd715104b6c80cc2e57c916da0834fd/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-07-22T12:29:28,360 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-07-22T12:29:28,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1076): writing seq id for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,365 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): Opened ccd715104b6c80cc2e57c916da0834fd; next sequenceid=58; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10339488160, jitterRate=-0.037060126662254333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-07-22T12:29:28,366 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(992): Region open journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:28,367 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., pid=75, masterSystemTime=1690028968339 2023-07-22T12:29:28,368 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:28,368 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(158): Opened backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:28,369 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=ccd715104b6c80cc2e57c916da0834fd, regionState=OPEN, openSeqNum=58, regionLocation=jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:28,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=75, resume processing ppid=74 2023-07-22T12:29:28,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=75, ppid=74, state=SUCCESS; OpenRegionProcedure ccd715104b6c80cc2e57c916da0834fd, server=jenkins-hbase3.apache.org,36719,1690028908802 in 183 msec 2023-07-22T12:29:28,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=74, resume processing ppid=73 2023-07-22T12:29:28,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=74, ppid=73, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=ccd715104b6c80cc2e57c916da0834fd, ASSIGN in 342 msec 2023-07-22T12:29:28,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1690028968374"}]},"ts":"1690028968374"} 2023-07-22T12:29:28,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-07-22T12:29:28,377 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-07-22T12:29:28,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=73, state=SUCCESS; EnableTableProcedure table=backup:system in 354 msec 2023-07-22T12:29:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-07-22T12:29:28,629 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-07-22T12:29:28,629 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-07-22T12:29:28,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-07-22T12:29:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-07-22T12:29:28,640 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-07-22T12:29:28,640 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-07-22T12:29:28,642 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1690028951525 2023-07-22T12:29:28,646 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:28,646 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:28,712 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.ResourceChecker(175): after: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=604 (was 583) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64849@0x1b17cb3b java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$494/139964844.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2efc7c1-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2efc7c1-shared-pool-27 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64849@0x5ec75b4b java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$494/139964844.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64849@0x5ec75b4b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1977020646_16 at /127.0.0.1:52690 [Receiving block BP-1147127663-172.31.12.81-1690028902390:blk_1073741901_1077] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x52854ce7-shared-pool-35 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,36719,1690028908802)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2efc7c1-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:36719-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64849@0x5ec75b4b-SendThread(127.0.0.1:64849) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-453854409_16 at /127.0.0.1:58742 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x52854ce7-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x52854ce7-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1147127663-172.31.12.81-1690028902390:blk_1073741901_1077, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x52854ce7-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2efc7c1-shared-pool-26 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1147127663-172.31.12.81-1690028902390:blk_1073741902_1078, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64849@0x1b17cb3b-SendThread(127.0.0.1:64849) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: hconnection-0x2efc7c1-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,36719,1690028908802' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1977020646_16 at /127.0.0.1:52706 [Receiving block BP-1147127663-172.31.12.81-1690028902390:blk_1073741902_1078] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x2efc7c1-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-453854409_16 at /127.0.0.1:58746 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x52854ce7-shared-pool-34 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-453854409_16 at /127.0.0.1:58690 [Waiting for operation #28] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64849@0x1b17cb3b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: (jenkins-hbase3.apache.org,43215,1690028905806)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=681 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=118 (was 156), ProcessCount=168 (was 169), AvailableMemoryMB=16854 (was 16980) 2023-07-22T12:29:28,713 WARN [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-07-22T12:29:28,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690028955861_default_test-1690028918084" 2023-07-22T12:29:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690028955861_default_test-1690028918084 2023-07-22T12:29:28,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1690028959982_ns2_test-16900289180841" 2023-07-22T12:29:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43215 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1690028959982_ns2_test-16900289180841 2023-07-22T12:29:28,733 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-07-22T12:29:28,733 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:28,734 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-07-22T12:29:28,734 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] util.JVMClusterUtil(257): Found active master hash=264597881, stopped=false 2023-07-22T12:29:28,735 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:29:28,737 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] procedure2.ProcedureExecutor(631): Stopping 2023-07-22T12:29:28,738 DEBUG [Listener at jenkins-hbase3.apache.org/36493 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:28,738 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,36719,1690028908802' ***** 2023-07-22T12:29:28,739 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-07-22T12:29:28,739 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:29:28,754 INFO [RS:0;jenkins-hbase3:36719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@14d0d1a8{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-07-22T12:29:28,760 INFO [RS:0;jenkins-hbase3:36719 {}] server.AbstractConnector(383): Stopped ServerConnector@63fc2aad{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:29:28,760 INFO [RS:0;jenkins-hbase3:36719 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:29:28,761 INFO [RS:0;jenkins-hbase3:36719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@433cc464{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:29:28,762 INFO [RS:0;jenkins-hbase3:36719 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@24084749{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.log.dir/,STOPPED} 2023-07-22T12:29:28,764 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HeapMemoryManager(220): Stopping 2023-07-22T12:29:28,764 INFO [RS:0;jenkins-hbase3:36719 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-07-22T12:29:28,764 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-07-22T12:29:28,764 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-07-22T12:29:28,764 INFO [RS:0;jenkins-hbase3:36719 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-07-22T12:29:28,765 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for ccd715104b6c80cc2e57c916da0834fd 2023-07-22T12:29:28,765 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for 2cc0bf07e1b2d5a0dc35be6598786b24 2023-07-22T12:29:28,765 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:29:28,765 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for 62c1d4e2655165c29f9a13c43a911f4c 2023-07-22T12:29:28,765 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for 0966504315f369581de1e5b13d64f65b 2023-07-22T12:29:28,765 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for 6705f3d0b146a5036498f0b2faf38255 2023-07-22T12:29:28,766 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for d465cc75c102ecbb2abf7437d3ee9ec0 2023-07-22T12:29:28,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ccd715104b6c80cc2e57c916da0834fd, disabling compactions & flushes 2023-07-22T12:29:28,766 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:28,766 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,36719,1690028908802 2023-07-22T12:29:28,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:28,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. after waiting 0 ms 2023-07-22T12:29:28,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:28,766 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing ccd715104b6c80cc2e57c916da0834fd 2/2 column families, dataSize=324 B heapSize=984 B 2023-07-22T12:29:28,766 DEBUG [RS:0;jenkins-hbase3:36719 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:28,767 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-07-22T12:29:28,767 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-07-22T12:29:28,767 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-07-22T12:29:28,767 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-07-22T12:29:28,767 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-07-22T12:29:28,767 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1583): Online Regions={ccd715104b6c80cc2e57c916da0834fd=backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd., 2cc0bf07e1b2d5a0dc35be6598786b24=hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24., d5a34dd2c7434d800baac09500ad347f=ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f., 62c1d4e2655165c29f9a13c43a911f4c=ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c., 1588230740=hbase:meta,,1.1588230740, 0966504315f369581de1e5b13d64f65b=ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b., 6705f3d0b146a5036498f0b2faf38255=backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255., d465cc75c102ecbb2abf7437d3ee9ec0=test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0.} 2023-07-22T12:29:28,768 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1609): Waiting on 0966504315f369581de1e5b13d64f65b, 1588230740, 2cc0bf07e1b2d5a0dc35be6598786b24, 62c1d4e2655165c29f9a13c43a911f4c, 6705f3d0b146a5036498f0b2faf38255, ccd715104b6c80cc2e57c916da0834fd, d465cc75c102ecbb2abf7437d3ee9ec0, d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:29:28,770 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-07-22T12:29:28,770 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-07-22T12:29:28,771 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-07-22T12:29:28,778 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-07-22T12:29:28,778 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-07-22T12:29:28,779 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=17.20 KB heapSize=30.02 KB 2023-07-22T12:29:28,795 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/12022e33be78439fafb242d0f8f64fd7 is 284, key is session:backup_1690028951525/session:context/1690028968641/Put/seqid=0 2023-07-22T12:29:28,795 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:29:28,810 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/.tmp/info/e1aa1567bb8a4febbd7c5a5d29dbcad1 is 159, key is ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c./info:regioninfo/1690028920517/Put/seqid=0 2023-07-22T12:29:28,817 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.80 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/.tmp/info/e1aa1567bb8a4febbd7c5a5d29dbcad1 2023-07-22T12:29:28,823 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e1aa1567bb8a4febbd7c5a5d29dbcad1 2023-07-22T12:29:28,840 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/.tmp/table/f3854515599549abb8ed0bc860cebe18 is 59, key is ns2:test-16900289180841/table:state/1690028920526/Put/seqid=0 2023-07-22T12:29:28,850 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/.tmp/table/f3854515599549abb8ed0bc860cebe18 2023-07-22T12:29:28,858 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f3854515599549abb8ed0bc860cebe18 2023-07-22T12:29:28,859 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/.tmp/info/e1aa1567bb8a4febbd7c5a5d29dbcad1 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/info/e1aa1567bb8a4febbd7c5a5d29dbcad1 2023-07-22T12:29:28,868 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/info/e1aa1567bb8a4febbd7c5a5d29dbcad1, entries=74, sequenceid=79, filesize=14.4 K 2023-07-22T12:29:28,869 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/.tmp/table/f3854515599549abb8ed0bc860cebe18 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/table/f3854515599549abb8ed0bc860cebe18 2023-07-22T12:29:28,876 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/table/f3854515599549abb8ed0bc860cebe18, entries=15, sequenceid=79, filesize=5.8 K 2023-07-22T12:29:28,877 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~17.20 KB/17610, heapSize ~29.73 KB/30448, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=79, compaction requested=false 2023-07-22T12:29:28,884 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e1aa1567bb8a4febbd7c5a5d29dbcad1 2023-07-22T12:29:28,889 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f3854515599549abb8ed0bc860cebe18 2023-07-22T12:29:28,900 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=1 2023-07-22T12:29:28,902 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:28,902 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-07-22T12:29:28,903 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-07-22T12:29:28,903 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-07-22T12:29:28,903 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-07-22T12:29:28,968 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1609): Waiting on 0966504315f369581de1e5b13d64f65b, 2cc0bf07e1b2d5a0dc35be6598786b24, 62c1d4e2655165c29f9a13c43a911f4c, 6705f3d0b146a5036498f0b2faf38255, ccd715104b6c80cc2e57c916da0834fd, d465cc75c102ecbb2abf7437d3ee9ec0, d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:29:29,062 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-07-22T12:29:29,062 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-07-22T12:29:29,169 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1609): Waiting on 0966504315f369581de1e5b13d64f65b, 2cc0bf07e1b2d5a0dc35be6598786b24, 62c1d4e2655165c29f9a13c43a911f4c, 6705f3d0b146a5036498f0b2faf38255, ccd715104b6c80cc2e57c916da0834fd, d465cc75c102ecbb2abf7437d3ee9ec0, d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:29:29,203 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/12022e33be78439fafb242d0f8f64fd7 2023-07-22T12:29:29,210 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 12022e33be78439fafb242d0f8f64fd7 2023-07-22T12:29:29,211 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/.tmp/session/12022e33be78439fafb242d0f8f64fd7 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/12022e33be78439fafb242d0f8f64fd7 2023-07-22T12:29:29,218 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/session/12022e33be78439fafb242d0f8f64fd7, entries=2, sequenceid=62, filesize=5.2 K 2023-07-22T12:29:29,218 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~712 B/712, currentSize=0 B/0 for ccd715104b6c80cc2e57c916da0834fd in 452ms, sequenceid=62, compaction requested=true 2023-07-22T12:29:29,219 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-07-22T12:29:29,219 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 0bafe9ca03c0427f87758e00587e8db0 2023-07-22T12:29:29,220 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f51b983ec13445eda6d7939681e1bf0a 2023-07-22T12:29:29,221 DEBUG [StoreFileCloser-ccd715104b6c80cc2e57c916da0834fd-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 12022e33be78439fafb242d0f8f64fd7 2023-07-22T12:29:29,225 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system/ccd715104b6c80cc2e57c916da0834fd/recovered.edits/65.seqid, newMaxSeqId=65, maxSeqId=57 2023-07-22T12:29:29,226 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,226 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:29,226 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ccd715104b6c80cc2e57c916da0834fd: 2023-07-22T12:29:29,226 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1690028924965.ccd715104b6c80cc2e57c916da0834fd. 2023-07-22T12:29:29,226 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2cc0bf07e1b2d5a0dc35be6598786b24, disabling compactions & flushes 2023-07-22T12:29:29,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:29:29,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:29:29,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. after waiting 0 ms 2023-07-22T12:29:29,227 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:29:29,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2cc0bf07e1b2d5a0dc35be6598786b24 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-07-22T12:29:29,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/.tmp/info/8fb99f6ffaea4886a3b24f2d0bfb3209 is 45, key is default/info:d/1690028913556/Put/seqid=0 2023-07-22T12:29:29,369 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1609): Waiting on 0966504315f369581de1e5b13d64f65b, 2cc0bf07e1b2d5a0dc35be6598786b24, 62c1d4e2655165c29f9a13c43a911f4c, 6705f3d0b146a5036498f0b2faf38255, d465cc75c102ecbb2abf7437d3ee9ec0, d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:29:29,569 DEBUG [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1609): Waiting on 0966504315f369581de1e5b13d64f65b, 2cc0bf07e1b2d5a0dc35be6598786b24, 62c1d4e2655165c29f9a13c43a911f4c, 6705f3d0b146a5036498f0b2faf38255, d465cc75c102ecbb2abf7437d3ee9ec0, d5a34dd2c7434d800baac09500ad347f 2023-07-22T12:29:29,648 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/.tmp/info/8fb99f6ffaea4886a3b24f2d0bfb3209 2023-07-22T12:29:29,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8fb99f6ffaea4886a3b24f2d0bfb3209 2023-07-22T12:29:29,655 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/.tmp/info/8fb99f6ffaea4886a3b24f2d0bfb3209 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/info/8fb99f6ffaea4886a3b24f2d0bfb3209 2023-07-22T12:29:29,661 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/info/8fb99f6ffaea4886a3b24f2d0bfb3209, entries=7, sequenceid=11, filesize=5.1 K 2023-07-22T12:29:29,662 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 2cc0bf07e1b2d5a0dc35be6598786b24 in 434ms, sequenceid=11, compaction requested=false 2023-07-22T12:29:29,664 DEBUG [StoreFileCloser-2cc0bf07e1b2d5a0dc35be6598786b24-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8fb99f6ffaea4886a3b24f2d0bfb3209 2023-07-22T12:29:29,668 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/hbase/namespace/2cc0bf07e1b2d5a0dc35be6598786b24/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-07-22T12:29:29,669 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,670 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:29:29,670 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2cc0bf07e1b2d5a0dc35be6598786b24: 2023-07-22T12:29:29,670 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1690028912183.2cc0bf07e1b2d5a0dc35be6598786b24. 2023-07-22T12:29:29,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d5a34dd2c7434d800baac09500ad347f, disabling compactions & flushes 2023-07-22T12:29:29,671 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:29:29,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:29:29,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. after waiting 0 ms 2023-07-22T12:29:29,671 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:29:29,674 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns3/test-16900289180842/d5a34dd2c7434d800baac09500ad347f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:29:29,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:29:29,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d5a34dd2c7434d800baac09500ad347f: 2023-07-22T12:29:29,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16900289180842,,1690028921525.d5a34dd2c7434d800baac09500ad347f. 2023-07-22T12:29:29,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 62c1d4e2655165c29f9a13c43a911f4c, disabling compactions & flushes 2023-07-22T12:29:29,676 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:29,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:29,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. after waiting 0 ms 2023-07-22T12:29:29,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:29,676 DEBUG [StoreFileCloser-62c1d4e2655165c29f9a13c43a911f4c-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 167b6fcefa344acaa760c35269fcc9b9 2023-07-22T12:29:29,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns2/test-16900289180841/62c1d4e2655165c29f9a13c43a911f4c/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:29:29,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,681 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:29,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 62c1d4e2655165c29f9a13c43a911f4c: 2023-07-22T12:29:29,681 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16900289180841,,1690028919322.62c1d4e2655165c29f9a13c43a911f4c. 2023-07-22T12:29:29,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0966504315f369581de1e5b13d64f65b, disabling compactions & flushes 2023-07-22T12:29:29,682 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:29:29,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:29:29,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. after waiting 0 ms 2023-07-22T12:29:29,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:29:29,685 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/ns4/test-16900289180843/0966504315f369581de1e5b13d64f65b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:29:29,685 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,686 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:29:29,686 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0966504315f369581de1e5b13d64f65b: 2023-07-22T12:29:29,686 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16900289180843,,1690028923648.0966504315f369581de1e5b13d64f65b. 2023-07-22T12:29:29,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 6705f3d0b146a5036498f0b2faf38255, disabling compactions & flushes 2023-07-22T12:29:29,687 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:29:29,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:29:29,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. after waiting 0 ms 2023-07-22T12:29:29,687 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:29:29,690 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/backup/system_bulk/6705f3d0b146a5036498f0b2faf38255/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-07-22T12:29:29,691 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,691 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:29:29,691 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 6705f3d0b146a5036498f0b2faf38255: 2023-07-22T12:29:29,691 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1690028925596.6705f3d0b146a5036498f0b2faf38255. 2023-07-22T12:29:29,691 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d465cc75c102ecbb2abf7437d3ee9ec0, disabling compactions & flushes 2023-07-22T12:29:29,691 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:29,691 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:29,692 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. after waiting 0 ms 2023-07-22T12:29:29,692 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:29,694 DEBUG [StoreFileCloser-d465cc75c102ecbb2abf7437d3ee9ec0-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: b4f8cdd8a5dc4c0d84fc2c28f1304547 2023-07-22T12:29:29,696 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/data/default/test-1690028918084/d465cc75c102ecbb2abf7437d3ee9ec0/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-07-22T12:29:29,697 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-07-22T12:29:29,698 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:29,698 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d465cc75c102ecbb2abf7437d3ee9ec0: 2023-07-22T12:29:29,698 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1690028918084,,1690028918567.d465cc75c102ecbb2abf7437d3ee9ec0. 2023-07-22T12:29:29,770 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,36719,1690028908802; all regions closed. 2023-07-22T12:29:29,778 DEBUG [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/oldWALs 2023-07-22T12:29:29,778 INFO [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36719%2C1690028908802.meta:.meta(num 1690028955692) 2023-07-22T12:29:29,786 DEBUG [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/oldWALs 2023-07-22T12:29:29,786 INFO [RS:0;jenkins-hbase3:36719 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36719%2C1690028908802:(num 1690028955710) 2023-07-22T12:29:29,786 DEBUG [RS:0;jenkins-hbase3:36719 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:29,787 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.LeaseManager(133): Closed leases 2023-07-22T12:29:29,787 INFO [RS:0;jenkins-hbase3:36719 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-07-22T12:29:29,787 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:29:29,789 INFO [RS:0;jenkins-hbase3:36719 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:36719 2023-07-22T12:29:29,797 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,36719,1690028908802] 2023-07-22T12:29:29,797 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,36719,1690028908802; numProcessing=1 2023-07-22T12:29:29,799 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,36719,1690028908802 expired; onlineServers=0 2023-07-22T12:29:29,799 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,43215,1690028905806' ***** 2023-07-22T12:29:29,799 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-07-22T12:29:29,799 DEBUG [M:0;jenkins-hbase3:43215 {}] ipc.AbstractRpcClient(194): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cc140b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-07-22T12:29:29,799 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-07-22T12:29:29,841 INFO [M:0;jenkins-hbase3:43215 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5e79011{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-07-22T12:29:29,842 INFO [M:0;jenkins-hbase3:43215 {}] server.AbstractConnector(383): Stopped ServerConnector@29f0e65e{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:29:29,842 INFO [M:0;jenkins-hbase3:43215 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-07-22T12:29:29,843 INFO [M:0;jenkins-hbase3:43215 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4f1ed2f7{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-07-22T12:29:29,843 INFO [M:0;jenkins-hbase3:43215 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5ac82cb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/hadoop.log.dir/,STOPPED} 2023-07-22T12:29:29,844 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43215,1690028905806 2023-07-22T12:29:29,844 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43215,1690028905806; all regions closed. 2023-07-22T12:29:29,844 DEBUG [M:0;jenkins-hbase3:43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:29,844 INFO [M:0;jenkins-hbase3:43215 {}] master.HMaster(1664): Stopping master jetty server 2023-07-22T12:29:29,845 INFO [M:0;jenkins-hbase3:43215 {}] server.AbstractConnector(383): Stopped ServerConnector@49910570{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-07-22T12:29:29,845 DEBUG [M:0;jenkins-hbase3:43215 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-07-22T12:29:29,845 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-07-22T12:29:29,845 INFO [M:0;jenkins-hbase3:43215 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-07-22T12:29:29,845 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690028910948 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1690028910948,5,FailOnTimeoutGroup] 2023-07-22T12:29:29,845 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690028910948 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1690028910948,5,FailOnTimeoutGroup] 2023-07-22T12:29:29,845 DEBUG [M:0;jenkins-hbase3:43215 {}] ipc.AbstractRpcClient(509): Stopping rpc client 2023-07-22T12:29:29,846 DEBUG [M:0;jenkins-hbase3:43215 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-07-22T12:29:29,846 INFO [M:0;jenkins-hbase3:43215 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-07-22T12:29:29,846 DEBUG [M:0;jenkins-hbase3:43215 {}] master.HMaster(1687): Stopping service threads 2023-07-22T12:29:29,846 INFO [M:0;jenkins-hbase3:43215 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-07-22T12:29:29,846 ERROR [M:0;jenkins-hbase3:43215 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-07-22T12:29:29,847 INFO [M:0;jenkins-hbase3:43215 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-07-22T12:29:29,847 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-07-22T12:29:29,848 WARN [M:0;jenkins-hbase3:43215 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-07-22T12:29:29,848 INFO [M:0;jenkins-hbase3:43215 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-07-22T12:29:29,848 INFO [M:0;jenkins-hbase3:43215 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-07-22T12:29:29,848 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-07-22T12:29:29,862 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:29:29,863 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:29:29,863 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-07-22T12:29:29,863 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:29:29,863 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=284.92 KB heapSize=342.04 KB 2023-07-22T12:29:29,875 DEBUG [M:0;jenkins-hbase3:43215 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bfea7fa7d2514e72b7843fd030e8d9cf is 88, key is hbase:meta,,1/info:sn/1690028911544/Put/seqid=0 2023-07-22T12:29:29,898 INFO [RS:0;jenkins-hbase3:36719 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,36719,1690028908802; zookeeper connection closed. 2023-07-22T12:29:29,899 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@721950c9 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@721950c9 2023-07-22T12:29:29,900 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-07-22T12:29:30,280 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bfea7fa7d2514e72b7843fd030e8d9cf 2023-07-22T12:29:30,285 DEBUG [M:0;jenkins-hbase3:43215 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bfea7fa7d2514e72b7843fd030e8d9cf 2023-07-22T12:29:30,301 DEBUG [M:0;jenkins-hbase3:43215 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c09870abc0a419284d4c0a4959130c3 is 1037, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1690028934933/Put/seqid=0 2023-07-22T12:29:30,706 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=284.33 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c09870abc0a419284d4c0a4959130c3 2023-07-22T12:29:30,713 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5c09870abc0a419284d4c0a4959130c3 2023-07-22T12:29:30,713 DEBUG [M:0;jenkins-hbase3:43215 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5c09870abc0a419284d4c0a4959130c3 2023-07-22T12:29:30,725 DEBUG [M:0;jenkins-hbase3:43215 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29dfe26527c747a5bed4addedbc8a5ed is 82, key is jenkins-hbase3.apache.org,36719,1690028908802/rs:state/1690028910979/Put/seqid=0 2023-07-22T12:29:30,786 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-07-22T12:29:31,129 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29dfe26527c747a5bed4addedbc8a5ed 2023-07-22T12:29:31,135 DEBUG [M:0;jenkins-hbase3:43215 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 29dfe26527c747a5bed4addedbc8a5ed 2023-07-22T12:29:31,136 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bfea7fa7d2514e72b7843fd030e8d9cf as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bfea7fa7d2514e72b7843fd030e8d9cf 2023-07-22T12:29:31,140 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bfea7fa7d2514e72b7843fd030e8d9cf, entries=8, sequenceid=679, filesize=5.5 K 2023-07-22T12:29:31,141 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c09870abc0a419284d4c0a4959130c3 as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5c09870abc0a419284d4c0a4959130c3 2023-07-22T12:29:31,146 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5c09870abc0a419284d4c0a4959130c3 2023-07-22T12:29:31,146 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5c09870abc0a419284d4c0a4959130c3, entries=75, sequenceid=679, filesize=23.2 K 2023-07-22T12:29:31,147 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/29dfe26527c747a5bed4addedbc8a5ed as hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29dfe26527c747a5bed4addedbc8a5ed 2023-07-22T12:29:31,152 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42225/user/jenkins/test-data/caa14d17-a617-9d4a-6751-5f2ca29e90f1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/29dfe26527c747a5bed4addedbc8a5ed, entries=1, sequenceid=679, filesize=5.1 K 2023-07-22T12:29:31,153 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(3022): Finished flush of dataSize ~284.92 KB/291762, heapSize ~341.74 KB/349944, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=679, compaction requested=false 2023-07-22T12:29:31,155 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bfea7fa7d2514e72b7843fd030e8d9cf 2023-07-22T12:29:31,156 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5c09870abc0a419284d4c0a4959130c3 2023-07-22T12:29:31,162 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 29dfe26527c747a5bed4addedbc8a5ed 2023-07-22T12:29:31,163 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-07-22T12:29:31,163 DEBUG [M:0;jenkins-hbase3:43215 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-07-22T12:29:31,166 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-07-22T12:29:31,166 INFO [M:0;jenkins-hbase3:43215 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-07-22T12:29:31,166 INFO [M:0;jenkins-hbase3:43215 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-07-22T12:29:31,167 INFO [M:0;jenkins-hbase3:43215 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:43215 2023-07-22T12:29:31,272 INFO [M:0;jenkins-hbase3:43215 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43215,1690028905806; zookeeper connection closed. 2023-07-22T12:29:31,274 WARN [Listener at jenkins-hbase3.apache.org/36493 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-07-22T12:29:31,278 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:29:31,384 WARN [BP-1147127663-172.31.12.81-1690028902390 heartbeating to localhost/127.0.0.1:42225 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-07-22T12:29:31,384 WARN [BP-1147127663-172.31.12.81-1690028902390 heartbeating to localhost/127.0.0.1:42225 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1147127663-172.31.12.81-1690028902390 (Datanode Uuid 7e1ba756-ba8c-40a6-a2c9-b8dff684b55b) service to localhost/127.0.0.1:42225 2023-07-22T12:29:31,386 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/dfs/data/data1/current/BP-1147127663-172.31.12.81-1690028902390 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:29:31,386 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/148c5c38-4d23-f2cc-ed8e-0fa745da81f6/cluster_ce445916-8aa0-71fc-55d3-20ec87f50f73/dfs/data/data2/current/BP-1147127663-172.31.12.81-1690028902390 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-07-22T12:29:31,418 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-07-22T12:29:31,568 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-07-22T12:29:31,568 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-07-22T12:29:31,577 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:29:31,692 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:29:31,801 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:29:31,802 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:29:31,906 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-07-22T12:29:31,910 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-07-22T12:29:31,912 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:29:31,916 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-07-22T12:29:32,018 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-07-22T12:29:32,018 INFO [Listener at jenkins-hbase3.apache.org/36493 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped