2023-08-10 06:24:56,668 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-08-10 06:24:56,683 main DEBUG Took 0.012747 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-08-10 06:24:56,683 main DEBUG PluginManager 'Core' found 129 plugins 2023-08-10 06:24:56,684 main DEBUG PluginManager 'Level' found 0 plugins 2023-08-10 06:24:56,685 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-08-10 06:24:56,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,697 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-08-10 06:24:56,715 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,718 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,718 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,720 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,720 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,721 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,722 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,723 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,723 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,724 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,724 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,726 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,726 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:24:56,727 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,727 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-08-10 06:24:56,729 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:24:56,731 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-08-10 06:24:56,733 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-08-10 06:24:56,734 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-08-10 06:24:56,735 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-08-10 06:24:56,736 main DEBUG PluginManager 'Converter' found 47 plugins 2023-08-10 06:24:56,750 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-08-10 06:24:56,753 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-08-10 06:24:56,754 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-08-10 06:24:56,754 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-08-10 06:24:56,754 main DEBUG createAppenders(={Console}) 2023-08-10 06:24:56,755 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-08-10 06:24:56,755 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-08-10 06:24:56,756 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-08-10 06:24:56,756 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-08-10 06:24:56,756 main DEBUG OutputStream closed 2023-08-10 06:24:56,757 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-08-10 06:24:56,757 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-08-10 06:24:56,757 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-08-10 06:24:56,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-08-10 06:24:56,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-08-10 06:24:56,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-08-10 06:24:56,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-08-10 06:24:56,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-08-10 06:24:56,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-08-10 06:24:56,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-08-10 06:24:56,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-08-10 06:24:56,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-08-10 06:24:56,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-08-10 06:24:56,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-08-10 06:24:56,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-08-10 06:24:56,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-08-10 06:24:56,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-08-10 06:24:56,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-08-10 06:24:56,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-08-10 06:24:56,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-08-10 06:24:56,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-08-10 06:24:56,823 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10 06:24:56,824 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-08-10 06:24:56,824 main DEBUG Shutdown hook enabled. Registering a new one. 2023-08-10 06:24:56,825 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-08-10T06:24:56,856 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupRepair timeout: 13 mins 2023-08-10 06:24:56,861 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-08-10 06:24:56,861 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10T06:24:57,231 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653 2023-08-10T06:24:57,237 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:24:57,240 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:24:57,241 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-08-10T06:24:57,245 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658, deleteOnExit=true 2023-08-10T06:24:57,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-08-10T06:24:57,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/test.cache.data in system properties and HBase conf 2023-08-10T06:24:57,246 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:24:57,246 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:24:57,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:24:57,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:24:57,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:24:57,361 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-08-10T06:24:57,859 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-08-10T06:24:57,864 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:24:57,865 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:24:57,866 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:24:57,866 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:24:57,866 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:24:57,866 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:24:57,867 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:24:57,867 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:24:57,867 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:24:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:24:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:24:57,868 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:24:57,869 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:24:57,871 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-08-10T06:24:58,356 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:24:58,360 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:24:58,678 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-08-10T06:24:58,822 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@5cc14ac1 via org.mortbay.log.Slf4jLog 2023-08-10T06:24:58,838 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:24:58,878 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:24:58,947 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir/Jetty_localhost_41877_hdfs____34ub6t/webapp 2023-08-10T06:24:59,130 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41877 2023-08-10T06:24:59,159 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:24:59,160 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:24:59,745 WARN [Listener at localhost/38181 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:24:59,812 WARN [Listener at localhost/38181 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-08-10T06:24:59,832 WARN [Listener at localhost/38181 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:24:59,838 INFO [Listener at localhost/38181 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:24:59,844 INFO [Listener at localhost/38181 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir/Jetty_localhost_35231_datanode____.vj1wh7/webapp 2023-08-10T06:24:59,945 INFO [Listener at localhost/38181 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35231 2023-08-10T06:25:00,277 WARN [Listener at localhost/38215 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:25:00,586 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-08-10T06:25:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x628724987e2493f0: Processing first storage report for DS-f225b02e-9d1e-4308-b858-604fee64b32b from datanode d4546392-d55d-4907-adb9-1e17c2760bef 2023-08-10T06:25:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x628724987e2493f0: from storage DS-f225b02e-9d1e-4308-b858-604fee64b32b node DatanodeRegistration(127.0.0.1:46437, datanodeUuid=d4546392-d55d-4907-adb9-1e17c2760bef, infoPort=37975, infoSecurePort=0, ipcPort=38215, storageInfo=lv=-57;cid=testClusterID;nsid=1662008432;c=1691648698437), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-08-10T06:25:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x628724987e2493f0: Processing first storage report for DS-d69436ee-a136-4f1a-933b-c8e1f9dd3e13 from datanode d4546392-d55d-4907-adb9-1e17c2760bef 2023-08-10T06:25:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x628724987e2493f0: from storage DS-d69436ee-a136-4f1a-933b-c8e1f9dd3e13 node DatanodeRegistration(127.0.0.1:46437, datanodeUuid=d4546392-d55d-4907-adb9-1e17c2760bef, infoPort=37975, infoSecurePort=0, ipcPort=38215, storageInfo=lv=-57;cid=testClusterID;nsid=1662008432;c=1691648698437), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-08-10T06:25:00,859 DEBUG [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653 2023-08-10T06:25:00,958 INFO [Listener at localhost/38215 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/zookeeper_0, clientPort=55613, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-08-10T06:25:00,975 INFO [Listener at localhost/38215 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=55613 2023-08-10T06:25:00,982 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:00,984 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:01,638 INFO [Listener at localhost/38215 {}] util.FSUtils(461): Created version file at hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 with version=8 2023-08-10T06:25:01,638 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/hbase-staging 2023-08-10T06:25:01,656 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-08-10T06:25:01,657 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-08-10T06:25:01,657 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-08-10T06:25:01,657 DEBUG [Listener at localhost/38215 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-08-10T06:25:01,886 DEBUG [Listener at localhost/38215 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-08-10T06:25:02,039 INFO [Listener at localhost/38215 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-08-10T06:25:02,713 INFO [Listener at localhost/38215 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:25:02,767 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:25:02,768 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:25:02,769 INFO [Listener at localhost/38215 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:25:02,769 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:25:02,769 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:25:02,926 INFO [Listener at localhost/38215 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:25:03,034 DEBUG [Listener at localhost/38215 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-08-10T06:25:03,040 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:25:03,103 DEBUG [Listener at localhost/38215 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 31974 (auto-detected) 2023-08-10T06:25:03,106 DEBUG [Listener at localhost/38215 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-08-10T06:25:03,165 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:43813 2023-08-10T06:25:03,180 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:03,182 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:03,202 INFO [Listener at localhost/38215 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:43813 connecting to ZooKeeper ensemble=127.0.0.1:55613 2023-08-10T06:25:03,252 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:438130x0, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:25:03,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:43813-0x10060e812670000 connected 2023-08-10T06:25:03,291 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:25:03,294 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:25:03,298 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:25:03,308 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43813 2023-08-10T06:25:03,309 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43813 2023-08-10T06:25:03,309 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43813 2023-08-10T06:25:03,309 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43813 2023-08-10T06:25:03,310 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43813 2023-08-10T06:25:03,364 INFO [Listener at localhost/38215 {}] log.Log(170): Logging initialized @7445ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-08-10T06:25:03,487 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:25:03,488 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:25:03,489 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:25:03,491 INFO [Listener at localhost/38215 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-08-10T06:25:03,491 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:25:03,491 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:25:03,495 INFO [Listener at localhost/38215 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:25:03,638 INFO [Listener at localhost/38215 {}] http.HttpServer(1219): Jetty bound to port 40727 2023-08-10T06:25:03,641 INFO [Listener at localhost/38215 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:25:03,685 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:25:03,690 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@488cf936{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:25:03,691 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:25:03,691 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@290f83bc{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:25:03,759 INFO [Listener at localhost/38215 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:25:03,774 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:25:03,775 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:25:03,777 INFO [Listener at localhost/38215 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:25:03,785 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:25:05,041 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@42679f83{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:25:05,054 INFO [Listener at localhost/38215 {}] server.AbstractConnector(333): Started ServerConnector@57e8e0d7{HTTP/1.1, (http/1.1)}{0.0.0.0:40727} 2023-08-10T06:25:05,054 INFO [Listener at localhost/38215 {}] server.Server(415): Started @9136ms 2023-08-10T06:25:05,058 INFO [Listener at localhost/38215 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0, hbase.cluster.distributed=false 2023-08-10T06:25:05,125 INFO [Listener at localhost/38215 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:25:05,125 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:25:05,125 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:25:05,125 INFO [Listener at localhost/38215 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:25:05,126 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:25:05,126 INFO [Listener at localhost/38215 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:25:05,134 INFO [Listener at localhost/38215 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:25:05,137 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:25:05,138 INFO [Listener at localhost/38215 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:44603 2023-08-10T06:25:05,141 INFO [Listener at localhost/38215 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-08-10T06:25:05,148 DEBUG [Listener at localhost/38215 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-08-10T06:25:05,149 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:05,153 INFO [Listener at localhost/38215 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:05,155 INFO [Listener at localhost/38215 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:44603 connecting to ZooKeeper ensemble=127.0.0.1:55613 2023-08-10T06:25:05,160 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:446030x0, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:25:05,161 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): regionserver:446030x0, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:25:05,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:44603-0x10060e812670001 connected 2023-08-10T06:25:05,162 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:25:05,163 DEBUG [Listener at localhost/38215 {}] zookeeper.ZKUtil(113): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:25:05,167 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44603 2023-08-10T06:25:05,168 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44603 2023-08-10T06:25:05,168 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44603 2023-08-10T06:25:05,169 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44603 2023-08-10T06:25:05,169 DEBUG [Listener at localhost/38215 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44603 2023-08-10T06:25:05,174 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:25:05,174 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:25:05,174 INFO [Listener at localhost/38215 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:25:05,175 INFO [Listener at localhost/38215 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-08-10T06:25:05,175 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:25:05,175 INFO [Listener at localhost/38215 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:25:05,176 INFO [Listener at localhost/38215 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:25:05,178 INFO [Listener at localhost/38215 {}] http.HttpServer(1219): Jetty bound to port 34755 2023-08-10T06:25:05,178 INFO [Listener at localhost/38215 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:25:05,183 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:25:05,183 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@16e118f3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:25:05,184 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:25:05,184 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@11566dea{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:25:05,197 INFO [Listener at localhost/38215 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:25:05,198 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:25:05,198 INFO [Listener at localhost/38215 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:25:05,198 INFO [Listener at localhost/38215 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-08-10T06:25:05,203 INFO [Listener at localhost/38215 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:25:05,208 INFO [Listener at localhost/38215 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@4969a0f2{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:25:05,209 INFO [Listener at localhost/38215 {}] server.AbstractConnector(333): Started ServerConnector@7207012b{HTTP/1.1, (http/1.1)}{0.0.0.0:34755} 2023-08-10T06:25:05,209 INFO [Listener at localhost/38215 {}] server.Server(415): Started @9290ms 2023-08-10T06:25:05,214 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:25:05,219 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@bfaf459{HTTP/1.1, (http/1.1)}{0.0.0.0:46665} 2023-08-10T06:25:05,219 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9300ms 2023-08-10T06:25:05,219 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:05,232 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:25:05,232 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:25:05,233 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:05,240 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:43813 2023-08-10T06:25:05,260 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:25:05,261 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:25:05,261 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:05,261 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:05,262 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:25:05,264 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,43813,1691648701870 from backup master directory 2023-08-10T06:25:05,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:25:05,268 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:05,268 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:25:05,269 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:25:05,269 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:25:05,270 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:05,275 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-08-10T06:25:05,277 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-08-10T06:25:05,390 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/hbase.id with ID: 9fce790a-71b9-43af-a8b2-61808e9aa5c3 2023-08-10T06:25:05,440 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:25:05,458 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:05,458 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:05,531 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:05,534 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-08-10T06:25:05,555 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-08-10T06:25:05,555 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-08-10T06:25:05,556 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:25:05,567 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:25:05,568 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:25:05,648 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store 2023-08-10T06:25:05,673 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-08-10T06:25:05,673 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:05,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:25:05,674 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:25:05,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:25:05,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:25:05,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:25:05,674 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:25:05,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:25:05,676 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/.initializing 2023-08-10T06:25:05,676 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/WALs/jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:05,686 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:25:05,701 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43813%2C1691648701870, suffix=, logDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/WALs/jenkins-hbase3.apache.org,43813,1691648701870, archiveDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/oldWALs, maxLogs=10 2023-08-10T06:25:05,733 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/WALs/jenkins-hbase3.apache.org,43813,1691648701870/jenkins-hbase3.apache.org%2C43813%2C1691648701870.1691648705708, exclude list is [], retry=0 2023-08-10T06:25:05,764 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:05,775 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:25:05,851 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/WALs/jenkins-hbase3.apache.org,43813,1691648701870/jenkins-hbase3.apache.org%2C43813%2C1691648701870.1691648705708 2023-08-10T06:25:05,851 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:05,852 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:05,853 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:05,859 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:05,860 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:05,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:05,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-08-10T06:25:05,999 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:06,003 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:06,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:06,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-08-10T06:25:06,008 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:06,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:06,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:06,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-08-10T06:25:06,019 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:06,020 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:06,020 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:06,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-08-10T06:25:06,023 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:06,024 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:06,029 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:06,031 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:06,050 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-08-10T06:25:06,057 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:25:06,063 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:06,064 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11659255680, jitterRate=0.08585280179977417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-08-10T06:25:06,073 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:25:06,075 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-08-10T06:25:06,113 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7caeaa58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:06,167 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-08-10T06:25:06,189 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-08-10T06:25:06,189 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-08-10T06:25:06,192 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-08-10T06:25:06,194 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-08-10T06:25:06,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 7 msec 2023-08-10T06:25:06,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-08-10T06:25:06,243 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-08-10T06:25:06,266 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-08-10T06:25:06,269 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-08-10T06:25:06,272 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-08-10T06:25:06,274 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-08-10T06:25:06,277 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-08-10T06:25:06,280 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-08-10T06:25:06,286 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-08-10T06:25:06,288 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-08-10T06:25:06,289 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-08-10T06:25:06,291 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-08-10T06:25:06,306 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-08-10T06:25:06,307 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-08-10T06:25:06,314 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:25:06,314 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:25:06,314 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,314 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,315 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,43813,1691648701870, sessionid=0x10060e812670000, setting cluster-up flag (Was=false) 2023-08-10T06:25:06,328 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-08-10T06:25:06,336 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,336 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,342 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-08-10T06:25:06,343 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:06,346 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,346 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,351 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-08-10T06:25:06,352 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:06,358 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,358 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:06,362 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-08-10T06:25:06,363 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:06,431 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:44603 2023-08-10T06:25:06,432 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(997): ClusterId : 9fce790a-71b9-43af-a8b2-61808e9aa5c3 2023-08-10T06:25:06,434 INFO [RS:0;jenkins-hbase3:44603 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-08-10T06:25:06,436 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-08-10T06:25:06,443 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-08-10T06:25:06,443 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-08-10T06:25:06,446 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-08-10T06:25:06,446 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-08-10T06:25:06,450 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-08-10T06:25:06,451 DEBUG [RS:0;jenkins-hbase3:44603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5851b394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:06,454 DEBUG [RS:0;jenkins-hbase3:44603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54bb2fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:25:06,461 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-08-10T06:25:06,461 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-08-10T06:25:06,461 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-08-10T06:25:06,465 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,43813,1691648701870 with isa=jenkins-hbase3.apache.org/172.31.12.81:44603, startcode=1691648705124 2023-08-10T06:25:06,491 DEBUG [RS:0;jenkins-hbase3:44603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:25:06,524 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-08-10T06:25:06,535 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-08-10T06:25:06,540 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-08-10T06:25:06,548 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,43813,1691648701870 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-08-10T06:25:06,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:25:06,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:25:06,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:25:06,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:25:06,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-08-10T06:25:06,553 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,554 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:25:06,554 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1691648736555 2023-08-10T06:25:06,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-08-10T06:25:06,563 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-08-10T06:25:06,564 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:25:06,565 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-08-10T06:25:06,570 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:06,570 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:25:06,578 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-08-10T06:25:06,581 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-08-10T06:25:06,583 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-08-10T06:25:06,584 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-08-10T06:25:06,584 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-08-10T06:25:06,586 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,587 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-08-10T06:25:06,589 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-08-10T06:25:06,590 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-08-10T06:25:06,593 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33543, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:25:06,597 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2b436acd to 127.0.0.1:55613 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:25:06,604 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43813 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:25:06,606 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775f27ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:06,606 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:25:06,609 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-08-10T06:25:06,610 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-08-10T06:25:06,614 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648706611,5,FailOnTimeoutGroup] 2023-08-10T06:25:06,614 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648706614,5,FailOnTimeoutGroup] 2023-08-10T06:25:06,614 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,614 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-08-10T06:25:06,616 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,616 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,636 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-08-10T06:25:06,636 WARN [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-08-10T06:25:06,737 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,43813,1691648701870 with isa=jenkins-hbase3.apache.org/172.31.12.81:44603, startcode=1691648705124 2023-08-10T06:25:06,742 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43813 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:06,754 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:06,754 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38181 2023-08-10T06:25:06,754 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=40727 2023-08-10T06:25:06,760 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-08-10T06:25:06,761 DEBUG [RS:0;jenkins-hbase3:44603 {}] zookeeper.ZKUtil(111): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:06,761 WARN [RS:0;jenkins-hbase3:44603 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:25:06,761 INFO [RS:0;jenkins-hbase3:44603 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:25:06,761 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:06,764 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,44603,1691648705124] 2023-08-10T06:25:06,788 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-08-10T06:25:06,797 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-08-10T06:25:06,811 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-08-10T06:25:06,815 INFO [RS:0;jenkins-hbase3:44603 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-08-10T06:25:06,815 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,816 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-08-10T06:25:06,825 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,826 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,826 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,826 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,826 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,826 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:25:06,827 DEBUG [RS:0;jenkins-hbase3:44603 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-08-10T06:25:06,828 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,828 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,828 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,829 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,829 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44603,1691648705124-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:25:06,844 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-08-10T06:25:06,847 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44603,1691648705124-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:06,861 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,44603,1691648705124 started 2023-08-10T06:25:06,862 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,44603,1691648705124, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:44603, sessionid=0x10060e812670001 2023-08-10T06:25:06,862 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-08-10T06:25:06,862 DEBUG [RS:0;jenkins-hbase3:44603 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:06,862 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44603,1691648705124' 2023-08-10T06:25:06,862 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-08-10T06:25:06,863 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-08-10T06:25:06,864 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-08-10T06:25:06,864 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-08-10T06:25:06,864 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44603,1691648705124' 2023-08-10T06:25:06,864 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:25:06,864 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:25:06,865 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-08-10T06:25:06,865 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-08-10T06:25:06,865 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-08-10T06:25:06,865 DEBUG [RS:0;jenkins-hbase3:44603 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:06,865 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44603,1691648705124' 2023-08-10T06:25:06,865 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-08-10T06:25:06,866 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-08-10T06:25:06,866 DEBUG [RS:0;jenkins-hbase3:44603 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-08-10T06:25:06,866 INFO [RS:0;jenkins-hbase3:44603 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-08-10T06:25:06,866 INFO [RS:0;jenkins-hbase3:44603 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-08-10T06:25:06,978 INFO [RS:0;jenkins-hbase3:44603 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-08-10T06:25:06,981 INFO [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44603%2C1691648705124, suffix=, logDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124, archiveDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/oldWALs, maxLogs=32 2023-08-10T06:25:06,988 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-08-10T06:25:06,988 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:06,996 DEBUG [RS:0;jenkins-hbase3:44603 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648706983, exclude list is [], retry=0 2023-08-10T06:25:07,005 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:07,005 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:07,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:25:07,010 INFO [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648706983 2023-08-10T06:25:07,011 DEBUG [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:07,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:25:07,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:07,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:25:07,015 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:25:07,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:07,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:25:07,018 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:25:07,018 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:07,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740 2023-08-10T06:25:07,021 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740 2023-08-10T06:25:07,024 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:25:07,027 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:25:07,031 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:07,032 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11070336000, jitterRate=0.031005382537841797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:25:07,034 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:25:07,034 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:25:07,035 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:25:07,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:25:07,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:25:07,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:25:07,037 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:25:07,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:25:07,040 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:25:07,040 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-08-10T06:25:07,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-08-10T06:25:07,065 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-08-10T06:25:07,070 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-08-10T06:25:07,222 DEBUG [jenkins-hbase3:43813 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-08-10T06:25:07,235 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:07,241 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44603,1691648705124, state=OPENING 2023-08-10T06:25:07,254 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-08-10T06:25:07,259 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:07,259 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:07,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:25:07,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:25:07,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:07,464 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:07,466 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-08-10T06:25:07,469 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36032, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-08-10T06:25:07,481 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-08-10T06:25:07,482 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:25:07,483 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-08-10T06:25:07,486 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta, suffix=.meta, logDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124, archiveDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/oldWALs, maxLogs=32 2023-08-10T06:25:07,498 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648707487.meta, exclude list is [], retry=0 2023-08-10T06:25:07,502 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:07,505 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648707487.meta 2023-08-10T06:25:07,506 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:07,506 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:07,511 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:07,512 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-08-10T06:25:07,528 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-08-10T06:25:07,528 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-08-10T06:25:07,533 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-08-10T06:25:07,533 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:07,533 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-08-10T06:25:07,533 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-08-10T06:25:07,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:25:07,537 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:25:07,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:07,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:25:07,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:25:07,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:07,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:25:07,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:25:07,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:25:07,543 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740 2023-08-10T06:25:07,546 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740 2023-08-10T06:25:07,548 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:25:07,550 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:25:07,552 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9756649760, jitterRate=-0.09134118258953094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:25:07,553 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:25:07,565 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1691648707454 2023-08-10T06:25:07,584 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-08-10T06:25:07,584 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-08-10T06:25:07,586 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:07,587 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44603,1691648705124, state=OPEN 2023-08-10T06:25:07,596 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:25:07,596 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:25:07,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:25:07,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:25:07,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-08-10T06:25:07,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44603,1691648705124 in 330 msec 2023-08-10T06:25:07,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-08-10T06:25:07,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 550 msec 2023-08-10T06:25:07,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.1880 sec 2023-08-10T06:25:07,615 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1691648707615, completionTime=-1 2023-08-10T06:25:07,615 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-08-10T06:25:07,615 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-08-10T06:25:07,805 DEBUG [hconnection-0x7eccc166-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:07,809 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36038, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:07,824 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-08-10T06:25:07,825 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1691648767825 2023-08-10T06:25:07,825 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1691648827825 2023-08-10T06:25:07,825 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 209 msec 2023-08-10T06:25:07,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43813,1691648701870-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:07,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43813,1691648701870-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:07,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43813,1691648701870-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:07,852 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:43813, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:07,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-08-10T06:25:07,859 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-08-10T06:25:07,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-08-10T06:25:07,877 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:25:07,887 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-08-10T06:25:07,893 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:07,896 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:07,898 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:07,935 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 13ebfcea9d0422eea8430a5d6dd6cb62, NAME => 'hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:07,955 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:07,955 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 13ebfcea9d0422eea8430a5d6dd6cb62, disabling compactions & flushes 2023-08-10T06:25:07,955 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:07,955 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:07,955 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. after waiting 0 ms 2023-08-10T06:25:07,955 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:07,955 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:07,955 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 13ebfcea9d0422eea8430a5d6dd6cb62: 2023-08-10T06:25:07,958 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:07,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1691648707960"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648707960"}]},"ts":"1691648707960"} 2023-08-10T06:25:08,013 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:08,016 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:08,022 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648708016"}]},"ts":"1691648708016"} 2023-08-10T06:25:08,029 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-08-10T06:25:08,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=13ebfcea9d0422eea8430a5d6dd6cb62, ASSIGN}] 2023-08-10T06:25:08,037 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=13ebfcea9d0422eea8430a5d6dd6cb62, ASSIGN 2023-08-10T06:25:08,039 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=13ebfcea9d0422eea8430a5d6dd6cb62, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:08,190 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=13ebfcea9d0422eea8430a5d6dd6cb62, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:08,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 13ebfcea9d0422eea8430a5d6dd6cb62, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:08,353 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:08,354 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 13ebfcea9d0422eea8430a5d6dd6cb62, NAME => 'hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:08,355 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:08,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:08,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,358 INFO [StoreOpener-13ebfcea9d0422eea8430a5d6dd6cb62-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,360 INFO [StoreOpener-13ebfcea9d0422eea8430a5d6dd6cb62-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13ebfcea9d0422eea8430a5d6dd6cb62 columnFamilyName info 2023-08-10T06:25:08,360 DEBUG [StoreOpener-13ebfcea9d0422eea8430a5d6dd6cb62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:08,361 INFO [StoreOpener-13ebfcea9d0422eea8430a5d6dd6cb62-1 {}] regionserver.HStore(324): Store=13ebfcea9d0422eea8430a5d6dd6cb62/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:08,362 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,363 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,366 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:25:08,369 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:08,370 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 13ebfcea9d0422eea8430a5d6dd6cb62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10903127040, jitterRate=0.01543283462524414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:25:08,371 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 13ebfcea9d0422eea8430a5d6dd6cb62: 2023-08-10T06:25:08,373 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62., pid=6, masterSystemTime=1691648708347 2023-08-10T06:25:08,376 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:08,376 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:25:08,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=13ebfcea9d0422eea8430a5d6dd6cb62, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:08,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-08-10T06:25:08,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 13ebfcea9d0422eea8430a5d6dd6cb62, server=jenkins-hbase3.apache.org,44603,1691648705124 in 188 msec 2023-08-10T06:25:08,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-08-10T06:25:08,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=13ebfcea9d0422eea8430a5d6dd6cb62, ASSIGN in 350 msec 2023-08-10T06:25:08,389 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:08,389 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648708389"}]},"ts":"1691648708389"} 2023-08-10T06:25:08,392 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-08-10T06:25:08,400 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:08,401 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-08-10T06:25:08,402 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44603-0x10060e812670001, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:08,402 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:08,402 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:25:08,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 522 msec 2023-08-10T06:25:08,458 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-08-10T06:25:08,485 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:08,490 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-08-10T06:25:08,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 47 msec 2023-08-10T06:25:08,506 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-08-10T06:25:08,523 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:08,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 22 msec 2023-08-10T06:25:08,546 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-08-10T06:25:08,549 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-08-10T06:25:08,549 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 3.279sec 2023-08-10T06:25:08,551 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-08-10T06:25:08,553 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-08-10T06:25:08,554 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-08-10T06:25:08,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-08-10T06:25:08,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-08-10T06:25:08,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43813,1691648701870-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:25:08,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43813,1691648701870-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-08-10T06:25:08,573 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-08-10T06:25:08,575 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-08-10T06:25:08,649 DEBUG [Listener at localhost/38215 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7bbbdb7a to 127.0.0.1:55613 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:25:08,655 DEBUG [Listener at localhost/38215 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb39958, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:08,661 DEBUG [Listener at localhost/38215 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-08-10T06:25:08,661 DEBUG [Listener at localhost/38215 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-08-10T06:25:08,680 DEBUG [hconnection-0x10b3a86c-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:08,696 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36044, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:08,723 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:25:08,723 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-08-10T06:25:08,723 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/test.cache.data in system properties and HBase conf 2023-08-10T06:25:08,723 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:25:08,723 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:25:08,723 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:25:08,724 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:25:08,725 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:25:08,725 INFO [Listener at localhost/38215 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-08-10T06:25:10,263 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:25:10,629 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:25:10,635 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40611_jobhistory____oc4lus/webapp 2023-08-10T06:25:10,827 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-08-10T06:25:10,827 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-08-10T06:25:10,827 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:25:10,831 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:25:10,918 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:11,054 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:11,398 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-08-10T06:25:11,443 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40611 2023-08-10T06:25:11,764 WARN [Listener at jenkins-hbase3.apache.org/41551 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:25:11,881 WARN [Listener at jenkins-hbase3.apache.org/41551 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:25:11,882 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:25:11,895 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39593_cluster____.wx4j8m/webapp 2023-08-10T06:25:11,978 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:25:11,978 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-08-10T06:25:11,978 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:25:11,979 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:25:11,987 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:12,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:25:12,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:12,042 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:25:12,042 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:12,045 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:25:12,045 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-08-10T06:25:12,046 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:25:12,046 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-08-10T06:25:12,046 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-08-10T06:25:12,046 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:12,227 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:12,323 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:12,331 INFO [Listener at jenkins-hbase3.apache.org/41551 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39593 2023-08-10T06:25:12,882 WARN [Listener at jenkins-hbase3.apache.org/45737 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:25:12,906 WARN [Listener at jenkins-hbase3.apache.org/45737 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:25:12,907 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:25:12,915 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44747_node____.k8t4rr/webapp 2023-08-10T06:25:12,970 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:25:12,970 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:25:12,970 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:25:12,971 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:25:12,982 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:13,014 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:13,080 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:25:13,140 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:13,166 INFO [Listener at jenkins-hbase3.apache.org/45737 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44747 2023-08-10T06:25:13,216 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-08-10T06:25:13,217 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-08-10T06:25:13,360 WARN [Listener at jenkins-hbase3.apache.org/44547 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:25:13,367 WARN [Listener at jenkins-hbase3.apache.org/44547 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:25:13,367 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:25:13,374 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40605_node____2oqv32/webapp 2023-08-10T06:25:13,427 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:25:13,428 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:25:13,428 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:25:13,428 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:25:13,439 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:13,455 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:13,518 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:25:13,525 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40605 2023-08-10T06:25:13,558 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-08-10T06:25:13,558 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:38181/backupUT 2023-08-10T06:25:13,569 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:13,574 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47606, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:13,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-08-10T06:25:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-08-10T06:25:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:25:13,616 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:13,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 35 msec 2023-08-10T06:25:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:25:13,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-08-10T06:25:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-08-10T06:25:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:25:13,720 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:13,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-08-10T06:25:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:25:13,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-08-10T06:25:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-08-10T06:25:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:25:13,830 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:13,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 15 msec 2023-08-10T06:25:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:25:13,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-08-10T06:25:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-08-10T06:25:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:25:13,939 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:13,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-08-10T06:25:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:25:14,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1691648713558', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1691648713558 2023-08-10T06:25:14,051 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1691648713558 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:14,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1691648713558" procId is: 13 2023-08-10T06:25:14,052 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:14,053 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1691648713558 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:25:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:25:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:25:14,487 INFO [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7643774d75fc83e3da00831f55e79730, NAME => 'test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691648713558', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:14,502 DEBUG [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:14,502 DEBUG [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1668): Closing 7643774d75fc83e3da00831f55e79730, disabling compactions & flushes 2023-08-10T06:25:14,502 INFO [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,502 DEBUG [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,502 DEBUG [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. after waiting 0 ms 2023-08-10T06:25:14,502 DEBUG [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,502 INFO [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1909): Closed test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,503 DEBUG [RegionOpenAndInit-test-1691648713558-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7643774d75fc83e3da00831f55e79730: 2023-08-10T06:25:14,504 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1691648713558 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:14,505 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691648714504"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648714504"}]},"ts":"1691648714504"} 2023-08-10T06:25:14,507 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:14,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1691648713558 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:14,509 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648713558","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648714509"}]},"ts":"1691648714509"} 2023-08-10T06:25:14,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648713558, state=ENABLING in hbase:meta 2023-08-10T06:25:14,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648713558, region=7643774d75fc83e3da00831f55e79730, ASSIGN}] 2023-08-10T06:25:14,525 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691648713558, region=7643774d75fc83e3da00831f55e79730, ASSIGN 2023-08-10T06:25:14,526 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691648713558, region=7643774d75fc83e3da00831f55e79730, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:25:14,678 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=7643774d75fc83e3da00831f55e79730, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:14,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 7643774d75fc83e3da00831f55e79730, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:14,844 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,844 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 7643774d75fc83e3da00831f55e79730, NAME => 'test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:14,844 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:14,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691648713558 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:14,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,848 INFO [StoreOpener-7643774d75fc83e3da00831f55e79730-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,851 INFO [StoreOpener-7643774d75fc83e3da00831f55e79730-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7643774d75fc83e3da00831f55e79730 columnFamilyName f 2023-08-10T06:25:14,851 DEBUG [StoreOpener-7643774d75fc83e3da00831f55e79730-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:14,852 INFO [StoreOpener-7643774d75fc83e3da00831f55e79730-1 {}] regionserver.HStore(324): Store=7643774d75fc83e3da00831f55e79730/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:14,854 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,854 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,858 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:14,865 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:14,866 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 7643774d75fc83e3da00831f55e79730; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11849800320, jitterRate=0.10359865427017212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:25:14,867 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 7643774d75fc83e3da00831f55e79730: 2023-08-10T06:25:14,869 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730., pid=15, masterSystemTime=1691648714834 2023-08-10T06:25:14,872 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,872 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:14,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=7643774d75fc83e3da00831f55e79730, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:14,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-08-10T06:25:14,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 7643774d75fc83e3da00831f55e79730, server=jenkins-hbase3.apache.org,44603,1691648705124 in 196 msec 2023-08-10T06:25:14,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-08-10T06:25:14,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1691648713558, region=7643774d75fc83e3da00831f55e79730, ASSIGN in 358 msec 2023-08-10T06:25:14,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1691648713558 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:14,886 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691648713558","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648714885"}]},"ts":"1691648714885"} 2023-08-10T06:25:14,888 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691648713558, state=ENABLED in hbase:meta 2023-08-10T06:25:14,891 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1691648713558 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:14,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1691648713558 in 843 msec 2023-08-10T06:25:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:25:15,165 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1691648713558, procId: 13 completed 2023-08-10T06:25:15,166 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4c2085aa to 127.0.0.1:55613 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:25:15,171 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a402fab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:15,173 DEBUG [hconnection-0x4b8b03fb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:15,176 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:51922, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:15,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44603 {}] regionserver.HRegion(8210): writing data to region test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:25:15,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16916487135581', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16916487135581 2023-08-10T06:25:15,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916487135581 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:15,280 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:15,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16916487135581" procId is: 16 2023-08-10T06:25:15,281 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16916487135581 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:25:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:25:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:25:15,700 INFO [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 880494b3d2545260caf6635e12cd4de8, NAME => 'ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916487135581', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:25:16,113 DEBUG [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:16,113 DEBUG [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1668): Closing 880494b3d2545260caf6635e12cd4de8, disabling compactions & flushes 2023-08-10T06:25:16,114 INFO [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,114 DEBUG [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,114 DEBUG [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. after waiting 0 ms 2023-08-10T06:25:16,114 DEBUG [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,114 INFO [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,114 DEBUG [RegionOpenAndInit-ns2:test-16916487135581-pool-0 {}] regionserver.HRegion(1622): Region close journal for 880494b3d2545260caf6635e12cd4de8: 2023-08-10T06:25:16,115 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16916487135581 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:16,116 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648716115"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648716115"}]},"ts":"1691648716115"} 2023-08-10T06:25:16,118 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:16,119 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16916487135581 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:16,119 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916487135581","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648716119"}]},"ts":"1691648716119"} 2023-08-10T06:25:16,121 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916487135581, state=ENABLING in hbase:meta 2023-08-10T06:25:16,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916487135581, region=880494b3d2545260caf6635e12cd4de8, ASSIGN}] 2023-08-10T06:25:16,128 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916487135581, region=880494b3d2545260caf6635e12cd4de8, ASSIGN 2023-08-10T06:25:16,128 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916487135581, region=880494b3d2545260caf6635e12cd4de8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:16,280 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=880494b3d2545260caf6635e12cd4de8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:16,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 880494b3d2545260caf6635e12cd4de8, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:25:16,439 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,439 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 880494b3d2545260caf6635e12cd4de8, NAME => 'ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:16,439 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:16,439 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916487135581 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,439 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:16,440 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,440 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,441 INFO [StoreOpener-880494b3d2545260caf6635e12cd4de8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,443 INFO [StoreOpener-880494b3d2545260caf6635e12cd4de8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 880494b3d2545260caf6635e12cd4de8 columnFamilyName f 2023-08-10T06:25:16,443 DEBUG [StoreOpener-880494b3d2545260caf6635e12cd4de8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:16,444 INFO [StoreOpener-880494b3d2545260caf6635e12cd4de8-1 {}] regionserver.HStore(324): Store=880494b3d2545260caf6635e12cd4de8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:16,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,446 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,449 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:16,456 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:16,457 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 880494b3d2545260caf6635e12cd4de8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9409376320, jitterRate=-0.12368354201316833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:25:16,458 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 880494b3d2545260caf6635e12cd4de8: 2023-08-10T06:25:16,460 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8., pid=18, masterSystemTime=1691648716434 2023-08-10T06:25:16,462 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,462 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:16,463 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=880494b3d2545260caf6635e12cd4de8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:16,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-08-10T06:25:16,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 880494b3d2545260caf6635e12cd4de8, server=jenkins-hbase3.apache.org,44603,1691648705124 in 184 msec 2023-08-10T06:25:16,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-08-10T06:25:16,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916487135581, region=880494b3d2545260caf6635e12cd4de8, ASSIGN in 344 msec 2023-08-10T06:25:16,473 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16916487135581 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:16,473 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916487135581","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648716473"}]},"ts":"1691648716473"} 2023-08-10T06:25:16,475 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916487135581, state=ENABLED in hbase:meta 2023-08-10T06:25:16,478 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916487135581 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:16,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16916487135581 in 1.2020 sec 2023-08-10T06:25:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:25:17,391 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16916487135581, procId: 16 completed 2023-08-10T06:25:17,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44603 {}] regionserver.HRegion(8210): writing data to region ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:25:17,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16916487135582', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16916487135582 2023-08-10T06:25:17,479 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916487135582 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:17,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16916487135582" procId is: 19 2023-08-10T06:25:17,480 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:17,481 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16916487135582 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:25:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:25:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:25:17,898 INFO [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a9ff049a133a7be329fe9b42f1efbce8, NAME => 'ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916487135582', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:25:18,315 DEBUG [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:18,315 DEBUG [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1668): Closing a9ff049a133a7be329fe9b42f1efbce8, disabling compactions & flushes 2023-08-10T06:25:18,315 INFO [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,315 DEBUG [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,315 DEBUG [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. after waiting 0 ms 2023-08-10T06:25:18,315 DEBUG [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,315 INFO [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,315 DEBUG [RegionOpenAndInit-ns3:test-16916487135582-pool-0 {}] regionserver.HRegion(1622): Region close journal for a9ff049a133a7be329fe9b42f1efbce8: 2023-08-10T06:25:18,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16916487135582 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:18,317 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648718317"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648718317"}]},"ts":"1691648718317"} 2023-08-10T06:25:18,319 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:18,320 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16916487135582 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:18,321 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916487135582","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648718320"}]},"ts":"1691648718320"} 2023-08-10T06:25:18,322 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916487135582, state=ENABLING in hbase:meta 2023-08-10T06:25:18,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916487135582, region=a9ff049a133a7be329fe9b42f1efbce8, ASSIGN}] 2023-08-10T06:25:18,330 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916487135582, region=a9ff049a133a7be329fe9b42f1efbce8, ASSIGN 2023-08-10T06:25:18,331 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916487135582, region=a9ff049a133a7be329fe9b42f1efbce8, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:18,482 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=a9ff049a133a7be329fe9b42f1efbce8, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:18,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure a9ff049a133a7be329fe9b42f1efbce8, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:25:18,642 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => a9ff049a133a7be329fe9b42f1efbce8, NAME => 'ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:18,643 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:18,643 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916487135582 a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,643 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:18,643 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,643 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,645 INFO [StoreOpener-a9ff049a133a7be329fe9b42f1efbce8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,646 INFO [StoreOpener-a9ff049a133a7be329fe9b42f1efbce8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9ff049a133a7be329fe9b42f1efbce8 columnFamilyName f 2023-08-10T06:25:18,647 DEBUG [StoreOpener-a9ff049a133a7be329fe9b42f1efbce8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:18,647 INFO [StoreOpener-a9ff049a133a7be329fe9b42f1efbce8-1 {}] regionserver.HStore(324): Store=a9ff049a133a7be329fe9b42f1efbce8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:18,648 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns3/test-16916487135582/a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns3/test-16916487135582/a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,652 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:25:18,655 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns3/test-16916487135582/a9ff049a133a7be329fe9b42f1efbce8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:18,656 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened a9ff049a133a7be329fe9b42f1efbce8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10216977120, jitterRate=-0.04846985638141632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:25:18,657 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for a9ff049a133a7be329fe9b42f1efbce8: 2023-08-10T06:25:18,658 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8., pid=21, masterSystemTime=1691648718637 2023-08-10T06:25:18,660 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,660 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:25:18,661 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=a9ff049a133a7be329fe9b42f1efbce8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:18,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-08-10T06:25:18,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure a9ff049a133a7be329fe9b42f1efbce8, server=jenkins-hbase3.apache.org,44603,1691648705124 in 178 msec 2023-08-10T06:25:18,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-08-10T06:25:18,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916487135582, region=a9ff049a133a7be329fe9b42f1efbce8, ASSIGN in 338 msec 2023-08-10T06:25:18,669 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16916487135582 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:18,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916487135582","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648718669"}]},"ts":"1691648718669"} 2023-08-10T06:25:18,671 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916487135582, state=ENABLED in hbase:meta 2023-08-10T06:25:18,673 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916487135582 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:18,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16916487135582 in 1.1970 sec 2023-08-10T06:25:19,218 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1691648713558' 2023-08-10T06:25:19,218 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16916487135581' 2023-08-10T06:25:19,219 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16916487135582' 2023-08-10T06:25:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:25:19,593 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16916487135582, procId: 19 completed 2023-08-10T06:25:19,593 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16916487135582 get assigned. Timeout = 60000ms 2023-08-10T06:25:19,595 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:25:19,599 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16916487135582 assigned to meta. Checking AM states. 2023-08-10T06:25:19,599 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:25:19,599 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16916487135582 assigned. 2023-08-10T06:25:19,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16916487135583', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16916487135583 2023-08-10T06:25:19,605 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916487135583 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:19,605 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:19,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16916487135583" procId is: 22 2023-08-10T06:25:19,606 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16916487135583 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:25:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:25:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:25:20,020 INFO [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 81197e323905f3bf93978d985f6c1d3d, NAME => 'ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16916487135583', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:20,033 DEBUG [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:20,033 DEBUG [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1668): Closing 81197e323905f3bf93978d985f6c1d3d, disabling compactions & flushes 2023-08-10T06:25:20,033 INFO [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,033 DEBUG [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,033 DEBUG [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. after waiting 0 ms 2023-08-10T06:25:20,033 DEBUG [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,033 INFO [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,033 DEBUG [RegionOpenAndInit-ns4:test-16916487135583-pool-0 {}] regionserver.HRegion(1622): Region close journal for 81197e323905f3bf93978d985f6c1d3d: 2023-08-10T06:25:20,035 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16916487135583 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:20,035 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691648720035"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648720035"}]},"ts":"1691648720035"} 2023-08-10T06:25:20,037 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:20,038 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16916487135583 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:20,038 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916487135583","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648720038"}]},"ts":"1691648720038"} 2023-08-10T06:25:20,040 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916487135583, state=ENABLING in hbase:meta 2023-08-10T06:25:20,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916487135583, region=81197e323905f3bf93978d985f6c1d3d, ASSIGN}] 2023-08-10T06:25:20,046 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916487135583, region=81197e323905f3bf93978d985f6c1d3d, ASSIGN 2023-08-10T06:25:20,047 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16916487135583, region=81197e323905f3bf93978d985f6c1d3d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:20,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=81197e323905f3bf93978d985f6c1d3d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:20,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 81197e323905f3bf93978d985f6c1d3d, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:20,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:25:20,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:25:20,359 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,360 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 81197e323905f3bf93978d985f6c1d3d, NAME => 'ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:20,360 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:20,360 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916487135583 81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:20,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,361 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,366 INFO [StoreOpener-81197e323905f3bf93978d985f6c1d3d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,373 INFO [StoreOpener-81197e323905f3bf93978d985f6c1d3d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81197e323905f3bf93978d985f6c1d3d columnFamilyName f 2023-08-10T06:25:20,374 DEBUG [StoreOpener-81197e323905f3bf93978d985f6c1d3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:20,377 INFO [StoreOpener-81197e323905f3bf93978d985f6c1d3d-1 {}] regionserver.HStore(324): Store=81197e323905f3bf93978d985f6c1d3d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:20,379 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns4/test-16916487135583/81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,379 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns4/test-16916487135583/81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,386 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:25:20,396 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns4/test-16916487135583/81197e323905f3bf93978d985f6c1d3d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:20,397 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 81197e323905f3bf93978d985f6c1d3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10090969600, jitterRate=-0.06020522117614746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:25:20,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 81197e323905f3bf93978d985f6c1d3d: 2023-08-10T06:25:20,406 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d., pid=24, masterSystemTime=1691648720354 2023-08-10T06:25:20,409 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,409 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:25:20,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=81197e323905f3bf93978d985f6c1d3d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:20,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-08-10T06:25:20,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 81197e323905f3bf93978d985f6c1d3d, server=jenkins-hbase3.apache.org,44603,1691648705124 in 213 msec 2023-08-10T06:25:20,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-08-10T06:25:20,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16916487135583, region=81197e323905f3bf93978d985f6c1d3d, ASSIGN in 373 msec 2023-08-10T06:25:20,424 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16916487135583 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:20,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916487135583","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648720424"}]},"ts":"1691648720424"} 2023-08-10T06:25:20,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916487135583, state=ENABLED in hbase:meta 2023-08-10T06:25:20,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916487135583 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:20,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16916487135583 in 828 msec 2023-08-10T06:25:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:25:20,714 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16916487135583, procId: 22 completed 2023-08-10T06:25:20,714 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16916487135583 get assigned. Timeout = 60000ms 2023-08-10T06:25:20,714 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:25:20,717 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16916487135583 assigned to meta. Checking AM states. 2023-08-10T06:25:20,718 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:25:20,718 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16916487135583 assigned. 2023-08-10T06:25:20,718 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4c2085aa to 127.0.0.1:55613 2023-08-10T06:25:20,718 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:20,760 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.ResourceChecker(147): before: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=135, ProcessCount=171, AvailableMemoryMB=2174 2023-08-10T06:25:20,760 WARN [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-08-10T06:25:20,764 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupRepair(57): Running stage 0 2023-08-10T06:25:20,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-08-10T06:25:20,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-08-10T06:25:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:25:20,794 DEBUG [Listener at localhost/38215-EventThread {}] zookeeper.ZKWatcher(604): master:43813-0x10060e812670000, quorum=127.0.0.1:55613, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:25:20,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 17 msec 2023-08-10T06:25:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:25:20,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-08-10T06:25:20,905 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:20,905 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:20,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-08-10T06:25:20,906 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:25:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:25:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:25:21,324 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:21,335 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:21,335 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing bc059d26009e8ee1c786aed65c192ae9, disabling compactions & flushes 2023-08-10T06:25:21,335 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,335 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,335 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. after waiting 0 ms 2023-08-10T06:25:21,336 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,336 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,336 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:21,338 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:21,338 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648721338"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648721338"}]},"ts":"1691648721338"} 2023-08-10T06:25:21,340 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:21,341 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:21,341 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648721341"}]},"ts":"1691648721341"} 2023-08-10T06:25:21,343 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:25:21,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN}] 2023-08-10T06:25:21,351 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN 2023-08-10T06:25:21,352 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:21,504 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:21,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:25:21,663 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,664 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:21,664 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:21,664 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:21,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,665 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,668 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,671 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName meta 2023-08-10T06:25:21,671 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:21,672 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:21,672 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,674 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName session 2023-08-10T06:25:21,674 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:21,674 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:21,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:25:21,679 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:21,681 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:21,682 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened bc059d26009e8ee1c786aed65c192ae9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11097467520, jitterRate=0.03353220224380493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:25:21,683 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:21,684 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., pid=28, masterSystemTime=1691648721659 2023-08-10T06:25:21,686 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,686 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:21,687 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:21,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-08-10T06:25:21,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 183 msec 2023-08-10T06:25:21,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-08-10T06:25:21,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN in 342 msec 2023-08-10T06:25:21,694 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:21,695 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648721694"}]},"ts":"1691648721694"} 2023-08-10T06:25:21,696 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:25:21,699 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:21,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 797 msec 2023-08-10T06:25:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=26 2023-08-10T06:25:22,014 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-08-10T06:25:22,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:25:22,034 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:22,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:25:22,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916487135581 2023-08-10T06:25:22,035 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916487135581 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-08-10T06:25:22,037 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916487135582 2023-08-10T06:25:22,037 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916487135582 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:22,037 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691648713558 2023-08-10T06:25:22,037 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1691648713558 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:22,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:25:22,038 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:22,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-08-10T06:25:22,038 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916487135583 2023-08-10T06:25:22,038 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916487135583 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:22,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:25:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:25:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:25:22,455 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 768720512dfecc14f1d87b27a2e78667, NAME => 'backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0 2023-08-10T06:25:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:25:22,867 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:22,867 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 768720512dfecc14f1d87b27a2e78667, disabling compactions & flushes 2023-08-10T06:25:22,867 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:22,867 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:22,867 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. after waiting 0 ms 2023-08-10T06:25:22,867 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:22,867 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:22,867 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 768720512dfecc14f1d87b27a2e78667: 2023-08-10T06:25:22,868 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:25:22,869 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691648722868"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648722868"}]},"ts":"1691648722868"} 2023-08-10T06:25:22,871 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:22,872 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:25:22,872 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648722872"}]},"ts":"1691648722872"} 2023-08-10T06:25:22,874 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-08-10T06:25:22,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=768720512dfecc14f1d87b27a2e78667, ASSIGN}] 2023-08-10T06:25:22,881 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=768720512dfecc14f1d87b27a2e78667, ASSIGN 2023-08-10T06:25:22,882 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=768720512dfecc14f1d87b27a2e78667, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44603,1691648705124; forceNewPlan=false, retain=false 2023-08-10T06:25:23,034 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=768720512dfecc14f1d87b27a2e78667, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:23,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 768720512dfecc14f1d87b27a2e78667, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:25:23,193 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:23,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 768720512dfecc14f1d87b27a2e78667, NAME => 'backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:23,193 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:23,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:23,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,195 INFO [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,197 INFO [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 768720512dfecc14f1d87b27a2e78667 columnFamilyName meta 2023-08-10T06:25:23,197 DEBUG [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:23,198 INFO [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] regionserver.HStore(324): Store=768720512dfecc14f1d87b27a2e78667/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:23,198 INFO [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,200 INFO [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 768720512dfecc14f1d87b27a2e78667 columnFamilyName session 2023-08-10T06:25:23,200 DEBUG [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:23,200 INFO [StoreOpener-768720512dfecc14f1d87b27a2e78667-1 {}] regionserver.HStore(324): Store=768720512dfecc14f1d87b27a2e78667/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:23,201 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system_bulk/768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,202 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system_bulk/768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,203 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:25:23,205 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:25:23,207 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system_bulk/768720512dfecc14f1d87b27a2e78667/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:25:23,208 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 768720512dfecc14f1d87b27a2e78667; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11284967840, jitterRate=0.05099453032016754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:25:23,208 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 768720512dfecc14f1d87b27a2e78667: 2023-08-10T06:25:23,209 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667., pid=31, masterSystemTime=1691648723188 2023-08-10T06:25:23,211 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:23,211 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:25:23,212 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=768720512dfecc14f1d87b27a2e78667, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:23,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=31, resume processing ppid=30 2023-08-10T06:25:23,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 768720512dfecc14f1d87b27a2e78667, server=jenkins-hbase3.apache.org,44603,1691648705124 in 178 msec 2023-08-10T06:25:23,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=30, resume processing ppid=29 2023-08-10T06:25:23,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=768720512dfecc14f1d87b27a2e78667, ASSIGN in 337 msec 2023-08-10T06:25:23,219 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:25:23,219 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648723219"}]},"ts":"1691648723219"} 2023-08-10T06:25:23,221 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-08-10T06:25:23,224 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:25:23,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1900 sec 2023-08-10T06:25:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=29 2023-08-10T06:25:24,147 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-08-10T06:25:24,194 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b8a19b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:24,195 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:24,197 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:24,201 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:24,203 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:24,260 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:25:24,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648724310 (current time:1691648724310). 2023-08-10T06:25:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:25:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b7d5c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:24,324 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40392, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:25:24,347 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:24,354 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:24,357 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:24,774 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:25:24,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9}] 2023-08-10T06:25:24,785 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:24,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44603 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-08-10T06:25:24,945 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:24,947 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=45 B heapSize=632 B 2023-08-10T06:25:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:25,019 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/830d693c18ed4cb39c7b64dc64bb4a0b is 49, key is activesession:/session:c/1691648724293/Put/seqid=0 2023-08-10T06:25:25,220 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16916487135583' 2023-08-10T06:25:25,220 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-08-10T06:25:25,221 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:25:25,438 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:25,502 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:25,505 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/830d693c18ed4cb39c7b64dc64bb4a0b as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:25,513 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, entries=1, sequenceid=5, filesize=4.9 K 2023-08-10T06:25:25,517 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 570ms, sequenceid=5, compaction requested=false 2023-08-10T06:25:25,519 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:25,519 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. for snapshot_backup_system completed. 2023-08-10T06:25:25,520 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:25:25,527 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:25:25,531 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:25:25,532 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b] hfiles 2023-08-10T06:25:25,532 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b for snapshot=snapshot_backup_system 2023-08-10T06:25:25,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:25:25,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:25,955 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-08-10T06:25:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster(3994): Remote procedure done, pid=33 2023-08-10T06:25:25,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:25,960 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:25,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-08-10T06:25:25,963 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:25:25,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 in 1.1830 sec 2023-08-10T06:25:25,965 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:25,968 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:25,968 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:25:25,970 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:25:26,395 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:26,410 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:26,410 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:25:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:26,845 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:26,845 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-08-10T06:25:26,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.5070 sec 2023-08-10T06:25:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=32 2023-08-10T06:25:28,458 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(115): Backup backup_1691648724234 started at 1691648728457. 2023-08-10T06:25:28,485 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(119): Backup session backup_1691648724234 has been started. Backup session finished. Status: FAILURE 2023-08-10T06:25:28,485 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:28,485 ERROR [Listener at jenkins-hbase3.apache.org/44547 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] ... 32 more 2023-08-10T06:25:28,492 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658fc23f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:28,497 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c427965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:28,497 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:28,499 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:28,504 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:28,505 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40396, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1691648724234,startts=1691648728457,failedts=1691648728526,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1691648724234 2023-08-10T06:25:28,526 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:25:28,528 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:25:28,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:25:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:25:28,573 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648728573"}]},"ts":"1691648728573"} 2023-08-10T06:25:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:25:28,575 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:25:28,578 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:25:28,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN}] 2023-08-10T06:25:28,582 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN 2023-08-10T06:25:28,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:28,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:25:28,747 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(119): Close bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:28,748 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1668): Closing bc059d26009e8ee1c786aed65c192ae9, disabling compactions & flushes 2023-08-10T06:25:28,748 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1690): Closing region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:28,748 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:28,748 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. after waiting 0 ms 2023-08-10T06:25:28,748 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:28,748 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=214 B heapSize=800 B 2023-08-10T06:25:28,756 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/a878bff1f9a049ba817b0dab5c8f2952 is 218, key is session:backup_1691648724234/session:context/1691648728483/Put/seqid=0 2023-08-10T06:25:28,771 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/a878bff1f9a049ba817b0dab5c8f2952 2023-08-10T06:25:28,780 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a878bff1f9a049ba817b0dab5c8f2952 2023-08-10T06:25:28,781 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/a878bff1f9a049ba817b0dab5c8f2952 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/a878bff1f9a049ba817b0dab5c8f2952 2023-08-10T06:25:28,789 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/a878bff1f9a049ba817b0dab5c8f2952, entries=1, sequenceid=9, filesize=5.2 K 2023-08-10T06:25:28,791 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 43ms, sequenceid=9, compaction requested=false 2023-08-10T06:25:28,793 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:28,794 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a878bff1f9a049ba817b0dab5c8f2952 2023-08-10T06:25:28,799 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-08-10T06:25:28,800 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:25:28,801 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1909): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:28,801 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1622): Region close journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:28,803 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(163): Closed bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:28,803 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSED 2023-08-10T06:25:28,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=36, resume processing ppid=35 2023-08-10T06:25:28,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, ppid=35, state=SUCCESS; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 220 msec 2023-08-10T06:25:28,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-08-10T06:25:28,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN in 226 msec 2023-08-10T06:25:28,809 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648728809"}]},"ts":"1691648728809"} 2023-08-10T06:25:28,811 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:25:28,813 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:25:28,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; DisableTableProcedure table=backup:system in 254 msec 2023-08-10T06:25:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:25:28,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648728890 (current time:1691648728890). 2023-08-10T06:25:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691648728889 VERSION not specified, setting to 2 2023-08-10T06:25:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ae83ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:28,900 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40402, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-08-10T06:25:28,912 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:25:28,913 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:28,915 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:28,925 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:25:28,927 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:25:28,927 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:25:28,927 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:28,927 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(327): No files under family: meta 2023-08-10T06:25:28,927 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:28,929 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/a878bff1f9a049ba817b0dab5c8f2952] hfiles 2023-08-10T06:25:28,929 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:28,929 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/a878bff1f9a049ba817b0dab5c8f2952 2023-08-10T06:25:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:25:29,338 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:25:29,340 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:29,341 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:29,341 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691648728889 2023-08-10T06:25:29,342 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648728889 2023-08-10T06:25:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:25:29,758 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:29,766 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:29,767 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648728889 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691648728889 2023-08-10T06:25:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:25:30,598 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:30,598 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-08-10T06:25:30,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648728889 table=backup:system type=FLUSH ttl=0 } in 1.6890 sec 2023-08-10T06:25:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:25:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=38, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648724310 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:25:31,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:25:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:25:31,068 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-08-10T06:25:31,069 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-08-10T06:25:31,070 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:31,075 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648724310 type: FLUSH version: 2 ttl: 0 2023-08-10T06:25:31,076 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system 2023-08-10T06:25:31,079 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:25:31,079 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,087 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:31,097 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/a878bff1f9a049ba817b0dab5c8f2952 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/a878bff1f9a049ba817b0dab5c8f2952 2023-08-10T06:25:31,097 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:31,098 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648724310 type: FLUSH version: 2 ttl: 0 2023-08-10T06:25:31,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648731099"}]},"ts":"9223372036854775807"} 2023-08-10T06:25:31,153 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:25:31,153 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:25:31,154 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648731100"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648731100"}]},"ts":"1691648731100"} 2023-08-10T06:25:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:25:31,156 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:31,156 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:25:31,156 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:25:31,156 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:25:31,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648724310 type: FLUSH version: 2 ttl: 0 ) in 118 msec 2023-08-10T06:25:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=38 2023-08-10T06:25:31,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648728889" 2023-08-10T06:25:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691648728889 2023-08-10T06:25:31,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:25:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=39, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:25:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:25:31,386 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:25:31,386 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648731386"}]},"ts":"1691648731386"} 2023-08-10T06:25:31,388 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:25:31,391 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:25:31,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN}] 2023-08-10T06:25:31,393 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN 2023-08-10T06:25:31,394 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:25:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:25:31,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:31,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:25:31,703 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:31,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:31,704 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:31,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:31,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,706 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,707 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName meta 2023-08-10T06:25:31,707 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:31,708 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:31,708 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,709 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName session 2023-08-10T06:25:31,709 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:31,718 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:31,718 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:31,719 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,721 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,723 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:25:31,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:31,726 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened bc059d26009e8ee1c786aed65c192ae9; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11289965280, jitterRate=0.05145995318889618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:25:31,727 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:31,727 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., pid=41, masterSystemTime=1691648731699 2023-08-10T06:25:31,729 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:31,729 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:31,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPEN, openSeqNum=13, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:31,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=41, resume processing ppid=40 2023-08-10T06:25:31,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 185 msec 2023-08-10T06:25:31,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-08-10T06:25:31,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN in 342 msec 2023-08-10T06:25:31,736 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648731736"}]},"ts":"1691648731736"} 2023-08-10T06:25:31,738 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:25:31,739 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:25:31,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, state=SUCCESS; EnableTableProcedure table=backup:system in 364 msec 2023-08-10T06:25:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:25:31,990 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:25:31,991 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:25:31,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:25:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:25:32,000 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:25:32,000 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-08-10T06:25:32,002 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1691648724234 2023-08-10T06:25:32,016 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:32,016 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:32,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-08-10T06:25:32,034 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-08-10T06:25:32,036 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupRepair(57): Running stage 1 2023-08-10T06:25:32,057 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de6b9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:32,058 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:32,059 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41798, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:32,063 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:32,065 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:32,108 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:25:32,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648732112 (current time:1691648732112). 2023-08-10T06:25:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:25:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55dd18ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:32,122 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47524, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-08-10T06:25:32,126 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:25:32,127 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:32,129 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:25:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:25:32,542 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:25:32,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9}] 2023-08-10T06:25:32,544 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:32,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44603 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-08-10T06:25:32,696 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:32,697 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-08-10T06:25:32,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/18c4b5d599b8468da12c1d470d05f807 is 284, key is session:backup_1691648724234/session:context/1691648732001/Put/seqid=0 2023-08-10T06:25:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:25:33,124 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:33,132 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:33,133 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/18c4b5d599b8468da12c1d470d05f807 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:33,139 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807, entries=2, sequenceid=18, filesize=5.2 K 2023-08-10T06:25:33,140 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 443ms, sequenceid=18, compaction requested=false 2023-08-10T06:25:33,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:25:33,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:33,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. for snapshot_backup_system completed. 2023-08-10T06:25:33,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:25:33,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:25:33,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:25:33,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807] hfiles 2023-08-10T06:25:33,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b for snapshot=snapshot_backup_system 2023-08-10T06:25:33,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 for snapshot=snapshot_backup_system 2023-08-10T06:25:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:25:33,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:33,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-08-10T06:25:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster(3994): Remote procedure done, pid=43 2023-08-10T06:25:33,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:33,556 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:33,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=43, resume processing ppid=42 2023-08-10T06:25:33,558 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:25:33,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 in 1.0150 sec 2023-08-10T06:25:33,559 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:33,560 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:33,560 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:25:33,561 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:25:33,580 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:33,588 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:33,589 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:25:34,009 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:34,009 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-08-10T06:25:34,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8850 sec 2023-08-10T06:25:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=42 2023-08-10T06:25:34,233 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(115): Backup backup_1691648732093 started at 1691648734233. 2023-08-10T06:25:34,235 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(119): Backup session backup_1691648732093 has been started. Backup session finished. Status: FAILURE 2023-08-10T06:25:34,271 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:34,272 ERROR [Listener at jenkins-hbase3.apache.org/44547 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] ... 32 more 2023-08-10T06:25:34,281 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622fcf3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:34,286 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c083ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:34,287 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:34,288 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41806, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:34,293 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:34,294 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:34,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties BackupId=backup_1691648732093,startts=1691648734233,failedts=1691648734312,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1691648732093 2023-08-10T06:25:34,312 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:25:34,313 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:25:34,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:25:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:25:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:25:34,328 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648734327"}]},"ts":"1691648734327"} 2023-08-10T06:25:34,329 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:25:34,331 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:25:34,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN}] 2023-08-10T06:25:34,336 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN 2023-08-10T06:25:34,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:34,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:25:34,492 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:34,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing bc059d26009e8ee1c786aed65c192ae9, disabling compactions & flushes 2023-08-10T06:25:34,492 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:34,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:34,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. after waiting 0 ms 2023-08-10T06:25:34,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:34,492 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=289 B heapSize=952 B 2023-08-10T06:25:34,507 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/c062ee8dd7a44fb5b8dc7d38881c1435 is 79, key is startcode:hdfs://localhost:38181/backupUT/meta:startcode/1691648734237/Put/seqid=0 2023-08-10T06:25:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:25:34,916 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=75 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/c062ee8dd7a44fb5b8dc7d38881c1435 2023-08-10T06:25:34,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c062ee8dd7a44fb5b8dc7d38881c1435 2023-08-10T06:25:34,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/95741c4d68eb4ce495349b135d51cae5 is 218, key is session:backup_1691648732093/session:context/1691648734234/Put/seqid=0 2023-08-10T06:25:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:25:34,941 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/95741c4d68eb4ce495349b135d51cae5 2023-08-10T06:25:34,948 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 95741c4d68eb4ce495349b135d51cae5 2023-08-10T06:25:34,949 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/c062ee8dd7a44fb5b8dc7d38881c1435 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/c062ee8dd7a44fb5b8dc7d38881c1435 2023-08-10T06:25:34,960 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/c062ee8dd7a44fb5b8dc7d38881c1435, entries=1, sequenceid=23, filesize=5.1 K 2023-08-10T06:25:34,961 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/95741c4d68eb4ce495349b135d51cae5 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/95741c4d68eb4ce495349b135d51cae5 2023-08-10T06:25:34,967 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/95741c4d68eb4ce495349b135d51cae5, entries=1, sequenceid=23, filesize=5.2 K 2023-08-10T06:25:34,968 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(3022): Finished flush of dataSize ~289 B/289, heapSize ~920 B/920, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 476ms, sequenceid=23, compaction requested=true 2023-08-10T06:25:34,970 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c062ee8dd7a44fb5b8dc7d38881c1435 2023-08-10T06:25:34,973 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:34,975 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:34,977 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 95741c4d68eb4ce495349b135d51cae5 2023-08-10T06:25:34,986 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=12 2023-08-10T06:25:34,987 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:25:34,988 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:34,988 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:34,992 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSED 2023-08-10T06:25:34,993 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:34,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=46, resume processing ppid=45 2023-08-10T06:25:34,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 656 msec 2023-08-10T06:25:35,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=45, resume processing ppid=44 2023-08-10T06:25:35,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN in 666 msec 2023-08-10T06:25:35,002 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648735001"}]},"ts":"1691648735001"} 2023-08-10T06:25:35,004 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:25:35,006 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:25:35,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, state=SUCCESS; DisableTableProcedure table=backup:system in 686 msec 2023-08-10T06:25:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=44 2023-08-10T06:25:35,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648735443 (current time:1691648735443). 2023-08-10T06:25:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691648735442 VERSION not specified, setting to 2 2023-08-10T06:25:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b8c045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:35,452 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47540, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-08-10T06:25:35,456 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:25:35,457 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:35,459 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:25:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:25:35,868 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:25:35,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:25:35,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:25:35,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:35,871 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/c062ee8dd7a44fb5b8dc7d38881c1435] hfiles 2023-08-10T06:25:35,871 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/c062ee8dd7a44fb5b8dc7d38881c1435 2023-08-10T06:25:35,871 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:35,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/95741c4d68eb4ce495349b135d51cae5] hfiles 2023-08-10T06:25:35,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:35,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:35,873 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/95741c4d68eb4ce495349b135d51cae5 2023-08-10T06:25:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:25:36,281 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:25:36,282 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:36,283 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:36,283 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691648735442 2023-08-10T06:25:36,284 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648735442 2023-08-10T06:25:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:25:36,695 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:36,703 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:36,704 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648735442 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691648735442 2023-08-10T06:25:37,524 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:37,524 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-08-10T06:25:37,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648735442 table=backup:system type=FLUSH ttl=0 } in 2.0700 sec 2023-08-10T06:25:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=47 2023-08-10T06:25:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=48, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648732112 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:25:37,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:25:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:25:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:25:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:25:37,985 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-08-10T06:25:37,986 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-08-10T06:25:37,987 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:37,991 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648732112 type: FLUSH version: 2 ttl: 0 2023-08-10T06:25:37,991 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system 2023-08-10T06:25:37,994 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:25:37,994 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,002 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/c062ee8dd7a44fb5b8dc7d38881c1435 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/c062ee8dd7a44fb5b8dc7d38881c1435 2023-08-10T06:25:38,002 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:38,006 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/95741c4d68eb4ce495349b135d51cae5 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/95741c4d68eb4ce495349b135d51cae5 2023-08-10T06:25:38,006 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:38,006 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648732112 type: FLUSH version: 2 ttl: 0 2023-08-10T06:25:38,008 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648738007"}]},"ts":"9223372036854775807"} 2023-08-10T06:25:38,010 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:25:38,010 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:25:38,010 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648738008"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648738008"}]},"ts":"1691648738008"} 2023-08-10T06:25:38,012 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:38,012 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:25:38,012 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:25:38,012 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:25:38,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648732112 type: FLUSH version: 2 ttl: 0 ) in 443 msec 2023-08-10T06:25:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=48 2023-08-10T06:25:38,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648735442" 2023-08-10T06:25:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691648735442 2023-08-10T06:25:38,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:25:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=49, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:25:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:25:38,187 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:25:38,187 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648738187"}]},"ts":"1691648738187"} 2023-08-10T06:25:38,189 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:25:38,191 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:25:38,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN}] 2023-08-10T06:25:38,193 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN 2023-08-10T06:25:38,194 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:25:38,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:25:38,345 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:38,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:38,457 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:25:38,460 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41814, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:25:38,502 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(130): Open backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:38,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7254): Opening region: {ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:38,503 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:38,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(888): Instantiated backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:38,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7296): checking encryption for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7299): checking classloading for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,505 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,506 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName meta 2023-08-10T06:25:38,506 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:38,506 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:38,506 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,507 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName session 2023-08-10T06:25:38,507 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:38,514 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:38,519 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:38,519 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:38,520 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,522 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:25:38,524 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1076): writing seq id for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:38,525 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): Opened bc059d26009e8ee1c786aed65c192ae9; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10052152480, jitterRate=-0.06382034718990326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:25:38,525 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(992): Region open journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:38,526 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., pid=51, masterSystemTime=1691648738499 2023-08-10T06:25:38,528 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:38,528 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(158): Opened backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:38,528 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPEN, openSeqNum=27, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:38,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=51, resume processing ppid=50 2023-08-10T06:25:38,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, ppid=50, state=SUCCESS; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 183 msec 2023-08-10T06:25:38,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=50, resume processing ppid=49 2023-08-10T06:25:38,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, ppid=49, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN in 340 msec 2023-08-10T06:25:38,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648738533"}]},"ts":"1691648738533"} 2023-08-10T06:25:38,535 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:25:38,536 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:25:38,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, state=SUCCESS; EnableTableProcedure table=backup:system in 354 msec 2023-08-10T06:25:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:25:38,790 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:25:38,790 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:25:38,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:25:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:25:38,800 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:25:38,800 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-08-10T06:25:38,802 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1691648732093 2023-08-10T06:25:38,807 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:38,808 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:38,842 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupRepair(57): Running stage 2 2023-08-10T06:25:38,864 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622734f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:38,865 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:38,867 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41816, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:38,871 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:38,874 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47542, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:38,921 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:25:38,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648738925 (current time:1691648738925). 2023-08-10T06:25:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:25:38,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ffefa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:38,940 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:47550, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:38,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-08-10T06:25:38,944 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:38,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:25:38,945 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:38,947 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:38,958 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:25:38,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9}] 2023-08-10T06:25:38,959 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:25:39,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44603 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-08-10T06:25:39,112 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:39,113 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-08-10T06:25:39,127 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/89c0c34b9bf2402a8cfa33fa1f4a6895 is 284, key is session:backup_1691648732093/session:context/1691648738801/Put/seqid=0 2023-08-10T06:25:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:25:39,532 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:39,539 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:39,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/89c0c34b9bf2402a8cfa33fa1f4a6895 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:39,547 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895, entries=2, sequenceid=32, filesize=5.2 K 2023-08-10T06:25:39,547 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 434ms, sequenceid=32, compaction requested=true 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. for snapshot_backup_system completed. 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:25:39,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895] hfiles 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b for snapshot=snapshot_backup_system 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 for snapshot=snapshot_backup_system 2023-08-10T06:25:39,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895 for snapshot=snapshot_backup_system 2023-08-10T06:25:39,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:39,555 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-08-10T06:25:39,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster(3994): Remote procedure done, pid=53 2023-08-10T06:25:39,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:39,557 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:39,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-08-10T06:25:39,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 in 599 msec 2023-08-10T06:25:39,559 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:25:39,560 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:39,560 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:39,560 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:25:39,561 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:25:39,971 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:39,980 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:39,980 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:25:40,415 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:40,415 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-08-10T06:25:40,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4730 sec 2023-08-10T06:25:40,490 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:25:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=52 2023-08-10T06:25:41,051 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(115): Backup backup_1691648738903 started at 1691648741051. 2023-08-10T06:25:41,053 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(119): Backup session backup_1691648738903 has been started. 2023-08-10T06:25:41,056 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-08-10T06:25:41,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:25:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:25:41,084 INFO [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:25:41,084 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:25:41,085 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:25:41,085 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:25:41,086 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:25:41,088 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,089 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:25:41,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:25:41,089 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:25:41,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:25:41,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:25:41,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:25:41,090 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:25:41,092 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:25:41,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:25:41,093 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:25:41,093 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:25:41,093 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:25:41,093 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:25:41,094 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44603,1691648705124' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:25:41,096 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:41,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:25:41,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:25:41,096 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:25:41,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:25:41,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:25:41,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:41,097 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:25:41,098 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44603,1691648705124' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:25:41,098 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:25:41,098 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5e87416b[Count = 0] remaining members to acquire global barrier 2023-08-10T06:25:41,098 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:41,100 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:41,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:41,101 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:25:41,101 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:25:41,101 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:25:41,102 DEBUG [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,102 INFO [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648706983 highest: 1691648706983 on jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,102 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44603%2C1691648705124:(num 1691648706983) roll requested 2023-08-10T06:25:41,114 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648741102, exclude list is [], retry=0 2023-08-10T06:25:41,118 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:41,122 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648706983 with entries=40, filesize=9.34 KB; new WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648741102 2023-08-10T06:25:41,123 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:41,123 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648706983 is not closed yet, will try archiving it next time 2023-08-10T06:25:41,123 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta:.meta(num 1691648707487) roll requested 2023-08-10T06:25:41,136 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648741123.meta, exclude list is [], retry=0 2023-08-10T06:25:41,140 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:41,142 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648707487.meta with entries=56, filesize=15.96 KB; new WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648741123.meta 2023-08-10T06:25:41,146 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:41,146 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648707487.meta is not closed yet, will try archiving it next time 2023-08-10T06:25:41,163 DEBUG [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-08-10T06:25:41,163 INFO [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648741102 on jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,165 DEBUG [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:41,166 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52000, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:25:41,186 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:25:41,186 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44603,1691648705124' in zk 2023-08-10T06:25:41,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,188 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:25:41,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:25:41,189 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:25:41,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:25:41,189 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:25:41,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:25:41,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:25:41,190 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:41,191 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,191 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:25:41,191 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:41,192 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,193 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44603,1691648705124': 2023-08-10T06:25:41,193 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44603,1691648705124' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:25:41,193 INFO [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:25:41,193 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:25:41,193 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:25:41,193 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:25:41,193 INFO [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:25:41,195 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:25:41,195 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:25:41,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:25:41,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:25:41,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:25:41,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:25:41,196 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:25:41,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:25:41,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:41,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:25:41,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:25:41,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:41,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:25:41,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:41,199 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,206 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:25:41,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:25:41,206 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:25:41,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:25:41,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:25:41,207 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:25:41,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:25:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done Backup session finished. Status: FAILURE 2023-08-10T06:25:41,318 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:41,319 ERROR [Listener at jenkins-hbase3.apache.org/44547 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] ... 32 more 2023-08-10T06:25:41,327 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a20a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:41,333 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d137e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:41,334 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:41,335 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52014, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:41,339 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:41,340 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52450, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1691648738903,startts=1691648741051,failedts=1691648741357,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1691648738903 2023-08-10T06:25:41,357 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:25:41,358 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:25:41,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:25:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:25:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:25:41,366 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648741366"}]},"ts":"1691648741366"} 2023-08-10T06:25:41,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:25:41,373 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:25:41,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN}] 2023-08-10T06:25:41,375 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN 2023-08-10T06:25:41,376 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:41,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:25:41,529 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:41,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing bc059d26009e8ee1c786aed65c192ae9, disabling compactions & flushes 2023-08-10T06:25:41,529 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:41,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:41,529 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. after waiting 0 ms 2023-08-10T06:25:41,530 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:41,530 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-08-10T06:25:41,544 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/1f1cb327379a4d57a814dd5740c33b6f is 116, key is rslogts:hdfs://localhost:38181/backupUT\x00jenkins-hbase3.apache.org:44603/meta:rs-log-ts/1691648741185/Put/seqid=0 2023-08-10T06:25:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:25:41,949 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/1f1cb327379a4d57a814dd5740c33b6f 2023-08-10T06:25:41,956 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1f1cb327379a4d57a814dd5740c33b6f 2023-08-10T06:25:41,960 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/57ebb2af7aa74aaa88de2eeeafecaecf is 218, key is session:backup_1691648738903/session:context/1691648741052/Put/seqid=0 2023-08-10T06:25:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:25:42,365 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/57ebb2af7aa74aaa88de2eeeafecaecf 2023-08-10T06:25:42,371 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 57ebb2af7aa74aaa88de2eeeafecaecf 2023-08-10T06:25:42,372 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/1f1cb327379a4d57a814dd5740c33b6f as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/1f1cb327379a4d57a814dd5740c33b6f 2023-08-10T06:25:42,378 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/1f1cb327379a4d57a814dd5740c33b6f, entries=2, sequenceid=38, filesize=5.3 K 2023-08-10T06:25:42,379 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/57ebb2af7aa74aaa88de2eeeafecaecf as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/57ebb2af7aa74aaa88de2eeeafecaecf 2023-08-10T06:25:42,385 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/57ebb2af7aa74aaa88de2eeeafecaecf, entries=1, sequenceid=38, filesize=5.2 K 2023-08-10T06:25:42,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 856ms, sequenceid=38, compaction requested=true 2023-08-10T06:25:42,388 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 1f1cb327379a4d57a814dd5740c33b6f 2023-08-10T06:25:42,389 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:42,391 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:42,393 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:42,394 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 57ebb2af7aa74aaa88de2eeeafecaecf 2023-08-10T06:25:42,398 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/recovered.edits/41.seqid, newMaxSeqId=41, maxSeqId=26 2023-08-10T06:25:42,399 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:25:42,399 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:42,399 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:42,401 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:42,401 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSED 2023-08-10T06:25:42,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=56, resume processing ppid=55 2023-08-10T06:25:42,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 1.0260 sec 2023-08-10T06:25:42,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-08-10T06:25:42,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN in 1.0300 sec 2023-08-10T06:25:42,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648742406"}]},"ts":"1691648742406"} 2023-08-10T06:25:42,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:25:42,409 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:25:42,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0470 sec 2023-08-10T06:25:42,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:25:42,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648742479 (current time:1691648742479). 2023-08-10T06:25:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691648742479 VERSION not specified, setting to 2 2023-08-10T06:25:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37010e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:42,492 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52462, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-08-10T06:25:42,496 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:25:42,497 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:42,499 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:25:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:25:42,907 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:25:42,908 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:25:42,908 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:25:42,908 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:42,910 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/1f1cb327379a4d57a814dd5740c33b6f] hfiles 2023-08-10T06:25:42,910 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/1f1cb327379a4d57a814dd5740c33b6f 2023-08-10T06:25:42,910 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:42,912 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/57ebb2af7aa74aaa88de2eeeafecaecf, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895] hfiles 2023-08-10T06:25:42,912 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:42,912 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/57ebb2af7aa74aaa88de2eeeafecaecf 2023-08-10T06:25:42,912 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:42,912 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:25:43,322 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:25:43,322 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:43,323 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:43,323 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691648742479 2023-08-10T06:25:43,324 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648742479 2023-08-10T06:25:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:25:43,735 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:43,743 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:43,744 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648742479 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691648742479 2023-08-10T06:25:44,562 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:44,563 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-08-10T06:25:44,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648742479 table=backup:system type=FLUSH ttl=0 } in 2.0700 sec 2023-08-10T06:25:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=57 2023-08-10T06:25:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=58, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648738925 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:25:44,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:25:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:25:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:25:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:25:45,025 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-08-10T06:25:45,025 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-08-10T06:25:45,026 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:45,029 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648738925 type: FLUSH version: 2 ttl: 0 2023-08-10T06:25:45,029 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system 2023-08-10T06:25:45,032 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:25:45,032 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,040 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/1f1cb327379a4d57a814dd5740c33b6f to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/1f1cb327379a4d57a814dd5740c33b6f 2023-08-10T06:25:45,040 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:45,043 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/57ebb2af7aa74aaa88de2eeeafecaecf to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/57ebb2af7aa74aaa88de2eeeafecaecf 2023-08-10T06:25:45,044 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:45,044 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648738925 type: FLUSH version: 2 ttl: 0 2023-08-10T06:25:45,045 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648745045"}]},"ts":"9223372036854775807"} 2023-08-10T06:25:45,047 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:25:45,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:25:45,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648745046"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648745046"}]},"ts":"1691648745046"} 2023-08-10T06:25:45,048 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:25:45,048 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:25:45,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:25:45,049 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:25:45,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648738925 type: FLUSH version: 2 ttl: 0 ) in 439 msec 2023-08-10T06:25:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=58 2023-08-10T06:25:45,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648742479" 2023-08-10T06:25:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691648742479 2023-08-10T06:25:45,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:25:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=59, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:25:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:25:45,226 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:25:45,226 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648745226"}]},"ts":"1691648745226"} 2023-08-10T06:25:45,228 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:25:45,230 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:25:45,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN}] 2023-08-10T06:25:45,232 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN 2023-08-10T06:25:45,233 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:25:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:25:45,385 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:45,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:25:45,541 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(130): Open backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:45,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7254): Opening region: {ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:25:45,542 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:25:45,542 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,542 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(888): Instantiated backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:25:45,542 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7296): checking encryption for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,542 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7299): checking classloading for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,543 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,544 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName meta 2023-08-10T06:25:45,545 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:45,545 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:45,545 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,546 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName session 2023-08-10T06:25:45,546 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:45,553 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:45,557 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:45,562 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:45,562 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:25:45,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,565 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,566 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:25:45,568 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1076): writing seq id for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:45,569 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): Opened bc059d26009e8ee1c786aed65c192ae9; next sequenceid=42; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11614513920, jitterRate=0.08168590068817139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:25:45,570 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(992): Region open journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:45,571 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., pid=61, masterSystemTime=1691648745538 2023-08-10T06:25:45,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(403): Add compact mark for store bc059d26009e8ee1c786aed65c192ae9:session, priority=-2147483648, current under compaction store size is 1 2023-08-10T06:25:45,575 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-08-10T06:25:45,576 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-08-10T06:25:45,577 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:45,577 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(158): Opened backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:45,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPEN, openSeqNum=42, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:45,579 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 15770 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-08-10T06:25:45,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=61, resume processing ppid=60 2023-08-10T06:25:45,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, ppid=60, state=SUCCESS; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 193 msec 2023-08-10T06:25:45,581 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.HStore(1534): bc059d26009e8ee1c786aed65c192ae9/session is initiating minor compaction (all files) 2023-08-10T06:25:45,581 INFO [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of bc059d26009e8ee1c786aed65c192ae9/session in backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:45,582 INFO [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895] into tmpdir=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp, totalSize=15.4 K 2023-08-10T06:25:45,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=60, resume processing ppid=59 2023-08-10T06:25:45,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, ppid=59, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN in 351 msec 2023-08-10T06:25:45,583 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] compactions.Compactor(223): Compacting 830d693c18ed4cb39c7b64dc64bb4a0b, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1691648724293 2023-08-10T06:25:45,583 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] compactions.Compactor(223): Compacting 18c4b5d599b8468da12c1d470d05f807, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1691648732001 2023-08-10T06:25:45,583 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648745583"}]},"ts":"1691648745583"} 2023-08-10T06:25:45,584 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] compactions.Compactor(223): Compacting 89c0c34b9bf2402a8cfa33fa1f4a6895, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1691648738801 2023-08-10T06:25:45,584 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:25:45,586 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:25:45,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, state=SUCCESS; EnableTableProcedure table=backup:system in 365 msec 2023-08-10T06:25:45,604 INFO [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bc059d26009e8ee1c786aed65c192ae9#session#compaction#8 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-08-10T06:25:45,605 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/8d744d1dd5cd4cd29116171f0ff17678 is 284, key is session:backup_1691648724234/session:context/1691648732001/Put/seqid=0 2023-08-10T06:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:25:45,829 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:25:45,829 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:25:45,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:25:45,838 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:25:45,838 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-08-10T06:25:45,840 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1691648738903 2023-08-10T06:25:45,845 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:45,845 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:45,901 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupRepair(57): Running stage 3 2023-08-10T06:25:45,919 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a28946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:45,920 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:45,921 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52024, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:45,925 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:45,927 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52468, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:45,967 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:25:45,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648745971 (current time:1691648745971). 2023-08-10T06:25:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:45,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:25:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b7055b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:45,981 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-08-10T06:25:45,985 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:25:45,985 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:45,987 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:46,018 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8d744d1dd5cd4cd29116171f0ff17678 2023-08-10T06:25:46,019 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/8d744d1dd5cd4cd29116171f0ff17678 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/8d744d1dd5cd4cd29116171f0ff17678 2023-08-10T06:25:46,035 INFO [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in bc059d26009e8ee1c786aed65c192ae9/session of bc059d26009e8ee1c786aed65c192ae9 into 8d744d1dd5cd4cd29116171f0ff17678(size=5.6 K), total size for store is 5.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-08-10T06:25:46,035 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:46,036 INFO [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., storeName=bc059d26009e8ee1c786aed65c192ae9/session, priority=13, startTime=1691648745574; duration=0sec 2023-08-10T06:25:46,036 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-08-10T06:25:46,036 DEBUG [RS:0;jenkins-hbase3:44603-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bc059d26009e8ee1c786aed65c192ae9:session 2023-08-10T06:25:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:25:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:25:46,394 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:25:46,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9}] 2023-08-10T06:25:46,396 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:46,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44603 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-08-10T06:25:46,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:46,548 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=369 B heapSize=1.08 KB 2023-08-10T06:25:46,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/836983030dee4b4c92ecdbbb02dcd61c is 284, key is session:backup_1691648738903/session:context/1691648745839/Put/seqid=0 2023-08-10T06:25:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:25:46,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=369 B at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:25:46,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:25:46,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/836983030dee4b4c92ecdbbb02dcd61c as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:25:46,971 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c, entries=2, sequenceid=48, filesize=5.2 K 2023-08-10T06:25:46,971 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~369 B/369, heapSize ~832 B/832, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 423ms, sequenceid=48, compaction requested=false 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. for snapshot_backup_system completed. 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/8d744d1dd5cd4cd29116171f0ff17678, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c] hfiles 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/8d744d1dd5cd4cd29116171f0ff17678 for snapshot=snapshot_backup_system 2023-08-10T06:25:46,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c for snapshot=snapshot_backup_system 2023-08-10T06:25:46,977 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:46,977 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-08-10T06:25:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster(3994): Remote procedure done, pid=63 2023-08-10T06:25:46,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:46,979 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:46,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-08-10T06:25:46,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure bc059d26009e8ee1c786aed65c192ae9 in 586 msec 2023-08-10T06:25:46,981 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:25:46,982 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:46,982 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:46,982 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:25:46,983 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:25:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:25:47,397 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:47,405 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:47,405 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:25:47,826 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:47,826 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-08-10T06:25:47,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8440 sec 2023-08-10T06:25:47,902 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:25:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=62 2023-08-10T06:25:48,091 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(115): Backup backup_1691648745954 started at 1691648748091. 2023-08-10T06:25:48,092 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(119): Backup session backup_1691648745954 has been started. 2023-08-10T06:25:48,094 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-08-10T06:25:48,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:25:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-08-10T06:25:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:25:48,095 INFO [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:25:48,095 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:25:48,097 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:25:48,097 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:25:48,097 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:25:48,100 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,100 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:25:48,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:25:48,100 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:25:48,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:25:48,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:25:48,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:25:48,101 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:25:48,101 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:25:48,101 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-08-10T06:25:48,101 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:25:48,101 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:25:48,101 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:25:48,102 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:25:48,102 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:25:48,102 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44603,1691648705124' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:25:48,104 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:48,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:25:48,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:25:48,105 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:25:48,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:25:48,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:25:48,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:48,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:25:48,107 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44603,1691648705124' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:25:48,107 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7a774b36[Count = 0] remaining members to acquire global barrier 2023-08-10T06:25:48,107 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:25:48,107 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:48,108 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:48,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:25:48,108 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:25:48,108 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:25:48,108 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:25:48,109 DEBUG [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,109 INFO [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691648741102 highest: 1691648741102 on jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,109 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44603%2C1691648705124:(num 1691648741102) roll requested 2023-08-10T06:25:48,121 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648748109, exclude list is [], retry=0 2023-08-10T06:25:48,125 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:48,127 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648741102 with entries=13, filesize=3.42 KB; new WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648748109 2023-08-10T06:25:48,127 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:48,127 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648741102 is not closed yet, will try archiving it next time 2023-08-10T06:25:48,127 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta:.meta(num 1691648741123) roll requested 2023-08-10T06:25:48,129 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.1691648741102 not finished, retry = 0 2023-08-10T06:25:48,139 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648748127.meta, exclude list is [], retry=0 2023-08-10T06:25:48,143 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK] 2023-08-10T06:25:48,144 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648741123.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648748127.meta 2023-08-10T06:25:48,144 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46437,DS-f225b02e-9d1e-4308-b858-604fee64b32b,DISK]] 2023-08-10T06:25:48,144 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648741123.meta is not closed yet, will try archiving it next time 2023-08-10T06:25:48,146 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/WALs/jenkins-hbase3.apache.org,44603,1691648705124/jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta.1691648741123.meta not finished, retry = 0 2023-08-10T06:25:48,149 DEBUG [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-08-10T06:25:48,149 INFO [rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691648748109 on jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,157 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:25:48,157 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44603,1691648705124' in zk 2023-08-10T06:25:48,159 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:25:48,159 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:25:48,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,159 DEBUG [member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:25:48,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:25:48,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:25:48,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:25:48,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:25:48,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:48,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:25:48,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:48,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44603,1691648705124': 2023-08-10T06:25:48,163 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44603,1691648705124' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:25:48,163 INFO [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:25:48,163 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:25:48,163 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:25:48,163 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:25:48,163 INFO [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:25:48,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:25:48,166 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:25:48,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:25:48,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:25:48,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:25:48,166 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:25:48,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:25:48,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:25:48,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:25:48,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:48,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:25:48,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:48,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:25:48,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:25:48,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:48,172 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:25:48,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:25:48,172 DEBUG [(jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:25:48,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:25:48,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:25:48,173 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:25:48,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:25:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:25:48,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } 2023-08-10T06:25:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648748276 (current time:1691648748276). 2023-08-10T06:25:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648748275_default_test-1691648713558 VERSION not specified, setting to 2 2023-08-10T06:25:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179dcb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:48,284 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52488, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } 2023-08-10T06:25:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:25:48,287 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:48,288 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:48,290 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:48,613 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-08-10T06:25:48,613 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-08-10T06:25:48,696 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:25:48,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7643774d75fc83e3da00831f55e79730}] 2023-08-10T06:25:48,698 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:48,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44603 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-08-10T06:25:48,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:48,851 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 7643774d75fc83e3da00831f55e79730 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:25:48,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/.tmp/f/e48b4016b9274b1b97275ba5bc817371 is 37, key is row10/f:q1/1691648715192/Put/seqid=0 2023-08-10T06:25:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:49,271 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/.tmp/f/e48b4016b9274b1b97275ba5bc817371 2023-08-10T06:25:49,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e48b4016b9274b1b97275ba5bc817371 2023-08-10T06:25:49,278 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/.tmp/f/e48b4016b9274b1b97275ba5bc817371 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/f/e48b4016b9274b1b97275ba5bc817371 2023-08-10T06:25:49,283 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/f/e48b4016b9274b1b97275ba5bc817371, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:25:49,284 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 7643774d75fc83e3da00831f55e79730 in 433ms, sequenceid=103, compaction requested=false 2023-08-10T06:25:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 7643774d75fc83e3da00831f55e79730: 2023-08-10T06:25:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. for snapshot_1691648748275_default_test-1691648713558 completed. 2023-08-10T06:25:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730.' region-info for snapshot=snapshot_1691648748275_default_test-1691648713558 2023-08-10T06:25:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:25:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/f/e48b4016b9274b1b97275ba5bc817371] hfiles 2023-08-10T06:25:49,286 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/f/e48b4016b9274b1b97275ba5bc817371 for snapshot=snapshot_1691648748275_default_test-1691648713558 2023-08-10T06:25:49,292 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:25:49,292 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-08-10T06:25:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster(3994): Remote procedure done, pid=65 2023-08-10T06:25:49,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648748275_default_test-1691648713558 on region 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:49,294 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7643774d75fc83e3da00831f55e79730 2023-08-10T06:25:49,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=65, resume processing ppid=64 2023-08-10T06:25:49,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 7643774d75fc83e3da00831f55e79730 in 599 msec 2023-08-10T06:25:49,297 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:25:49,298 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:49,298 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:49,298 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648748275_default_test-1691648713558 2023-08-10T06:25:49,299 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_1691648748275_default_test-1691648713558 2023-08-10T06:25:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:49,711 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:49,717 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:49,717 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_1691648748275_default_test-1691648713558 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/snapshot_1691648748275_default_test-1691648713558 2023-08-10T06:25:50,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:50,535 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:50,535 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:25:50,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1691648748275_default_test-1691648713558 table=test-1691648713558 type=FLUSH ttl=0 } in 2.2500 sec 2023-08-10T06:25:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:25:52,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } 2023-08-10T06:25:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648752394 (current time:1691648752394). 2023-08-10T06:25:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691648752394_ns2_test-16916487135581 VERSION not specified, setting to 2 2023-08-10T06:25:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@307f0398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:52,404 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } 2023-08-10T06:25:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-08-10T06:25:52,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:25:52,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:52,410 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:25:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:25:52,816 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:25:52,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 880494b3d2545260caf6635e12cd4de8}] 2023-08-10T06:25:52,820 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:52,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44603 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-08-10T06:25:52,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:52,972 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing 880494b3d2545260caf6635e12cd4de8 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:25:52,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/.tmp/f/46b9360ac0ee400d972892288374a1c2 is 37, key is row10/f:q1/1691648717405/Put/seqid=0 2023-08-10T06:25:53,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:25:53,391 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/.tmp/f/46b9360ac0ee400d972892288374a1c2 2023-08-10T06:25:53,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 46b9360ac0ee400d972892288374a1c2 2023-08-10T06:25:53,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/.tmp/f/46b9360ac0ee400d972892288374a1c2 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/f/46b9360ac0ee400d972892288374a1c2 2023-08-10T06:25:53,403 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/f/46b9360ac0ee400d972892288374a1c2, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:25:53,404 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 880494b3d2545260caf6635e12cd4de8 in 432ms, sequenceid=103, compaction requested=false 2023-08-10T06:25:53,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 880494b3d2545260caf6635e12cd4de8: 2023-08-10T06:25:53,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. for snapshot_1691648752394_ns2_test-16916487135581 completed. 2023-08-10T06:25:53,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8.' region-info for snapshot=snapshot_1691648752394_ns2_test-16916487135581 2023-08-10T06:25:53,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:25:53,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/f/46b9360ac0ee400d972892288374a1c2] hfiles 2023-08-10T06:25:53,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/f/46b9360ac0ee400d972892288374a1c2 for snapshot=snapshot_1691648752394_ns2_test-16916487135581 2023-08-10T06:25:53,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:25:53,811 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:25:53,811 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-08-10T06:25:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster(3994): Remote procedure done, pid=67 2023-08-10T06:25:53,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691648752394_ns2_test-16916487135581 on region 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:53,814 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:25:53,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=67, resume processing ppid=66 2023-08-10T06:25:53,816 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:25:53,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure 880494b3d2545260caf6635e12cd4de8 in 999 msec 2023-08-10T06:25:53,817 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:53,818 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:53,818 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691648752394_ns2_test-16916487135581 2023-08-10T06:25:53,818 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_1691648752394_ns2_test-16916487135581 2023-08-10T06:25:54,230 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:54,235 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:54,235 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/snapshot_1691648752394_ns2_test-16916487135581 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/snapshot_1691648752394_ns2_test-16916487135581 2023-08-10T06:25:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:25:55,052 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:55,052 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-08-10T06:25:55,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1691648752394_ns2_test-16916487135581 table=ns2:test-16916487135581 type=FLUSH ttl=0 } in 2.6470 sec 2023-08-10T06:25:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 Backup session finished. Status: FAILURE 2023-08-10T06:25:56,515 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:56,515 ERROR [Listener at jenkins-hbase3.apache.org/44547 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.runBackupAndFailAtStageWithRestore(TestBackupRepair.java:70) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupRepair.testFullBackupWithFailuresAndRestore(TestBackupRepair.java:58) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] ... 32 more 2023-08-10T06:25:56,523 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5561998c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:56,529 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf3a8f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:56,529 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:25:56,530 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:25:56,536 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:56,538 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService BackupId=backup_1691648745954,startts=1691648748091,failedts=1691648756556,failedphase=REQUEST,failedmessage=REPAIR status: repaired after failure: backup_1691648745954 2023-08-10T06:25:56,556 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:25:56,557 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:25:56,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:25:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:25:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:25:56,570 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648756569"}]},"ts":"1691648756569"} 2023-08-10T06:25:56,571 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:25:56,573 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:25:56,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN}] 2023-08-10T06:25:56,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN 2023-08-10T06:25:56,575 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:25:56,577 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:25:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:25:56,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:56,728 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing bc059d26009e8ee1c786aed65c192ae9, disabling compactions & flushes 2023-08-10T06:25:56,728 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:56,729 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:56,729 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. after waiting 0 ms 2023-08-10T06:25:56,729 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:56,729 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-08-10T06:25:56,742 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/799f23d6b78042de88c6f772a308daa3 is 116, key is rslogts:hdfs://localhost:38181/backupUT\x00jenkins-hbase3.apache.org:44603/meta:rs-log-ts/1691648748156/Put/seqid=0 2023-08-10T06:25:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:25:57,147 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/799f23d6b78042de88c6f772a308daa3 2023-08-10T06:25:57,154 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 799f23d6b78042de88c6f772a308daa3 2023-08-10T06:25:57,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/266080e572ea4022acd13bcdab5cae71 is 218, key is session:backup_1691648745954/session:context/1691648748091/Put/seqid=0 2023-08-10T06:25:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:25:57,562 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/266080e572ea4022acd13bcdab5cae71 2023-08-10T06:25:57,568 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 266080e572ea4022acd13bcdab5cae71 2023-08-10T06:25:57,569 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/meta/799f23d6b78042de88c6f772a308daa3 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/799f23d6b78042de88c6f772a308daa3 2023-08-10T06:25:57,576 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/799f23d6b78042de88c6f772a308daa3, entries=2, sequenceid=54, filesize=5.3 K 2023-08-10T06:25:57,577 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/266080e572ea4022acd13bcdab5cae71 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/266080e572ea4022acd13bcdab5cae71 2023-08-10T06:25:57,582 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/266080e572ea4022acd13bcdab5cae71, entries=1, sequenceid=54, filesize=5.2 K 2023-08-10T06:25:57,583 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 854ms, sequenceid=54, compaction requested=true 2023-08-10T06:25:57,585 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 799f23d6b78042de88c6f772a308daa3 2023-08-10T06:25:57,586 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:57,588 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:57,590 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:57,591 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895] to archive 2023-08-10T06:25:57,592 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-08-10T06:25:57,593 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:57,595 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/830d693c18ed4cb39c7b64dc64bb4a0b 2023-08-10T06:25:57,596 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:57,597 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/18c4b5d599b8468da12c1d470d05f807 2023-08-10T06:25:57,597 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:57,598 DEBUG [StoreCloser-backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/89c0c34b9bf2402a8cfa33fa1f4a6895 2023-08-10T06:25:57,627 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8d744d1dd5cd4cd29116171f0ff17678 2023-08-10T06:25:57,629 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:25:57,630 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 266080e572ea4022acd13bcdab5cae71 2023-08-10T06:25:57,635 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/recovered.edits/57.seqid, newMaxSeqId=57, maxSeqId=41 2023-08-10T06:25:57,635 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:25:57,636 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:25:57,636 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:25:57,638 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:25:57,638 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=CLOSED 2023-08-10T06:25:57,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=70, resume processing ppid=69 2023-08-10T06:25:57,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 1.0630 sec 2023-08-10T06:25:57,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=69, resume processing ppid=68 2023-08-10T06:25:57,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, UNASSIGN in 1.0680 sec 2023-08-10T06:25:57,643 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648757642"}]},"ts":"1691648757642"} 2023-08-10T06:25:57,644 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:25:57,645 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:25:57,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=68, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0800 sec 2023-08-10T06:25:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=68 2023-08-10T06:25:57,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691648757686 (current time:1691648757686). 2023-08-10T06:25:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:25:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691648757685 VERSION not specified, setting to 2 2023-08-10T06:25:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4d33fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:25:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:25:57,696 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41158, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:25:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:25:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:25:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=71, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:25:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-08-10T06:25:57,699 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:25:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:25:57,700 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:25:57,701 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:25:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:25:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:25:58,107 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:25:58,108 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:25:58,108 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:25:58,108 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:58,109 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/799f23d6b78042de88c6f772a308daa3] hfiles 2023-08-10T06:25:58,109 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/799f23d6b78042de88c6f772a308daa3 2023-08-10T06:25:58,109 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:25:58,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/266080e572ea4022acd13bcdab5cae71, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/8d744d1dd5cd4cd29116171f0ff17678] hfiles 2023-08-10T06:25:58,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/266080e572ea4022acd13bcdab5cae71 2023-08-10T06:25:58,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:25:58,111 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/8d744d1dd5cd4cd29116171f0ff17678 2023-08-10T06:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:25:58,517 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:25:58,519 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:25:58,519 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:25:58,519 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691648757685 2023-08-10T06:25:58,520 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648757685 2023-08-10T06:25:58,553 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:25:58,562 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:25:58,563 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691648757685 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691648757685 2023-08-10T06:25:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:25:59,383 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:25:59,383 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-08-10T06:25:59,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691648757685 table=backup:system type=FLUSH ttl=0 } in 1.6860 sec 2023-08-10T06:25:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:25:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=72, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648745971 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:25:59,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:25:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:25:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:26:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:26:00,227 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-08-10T06:26:00,227 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-08-10T06:26:00,229 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:26:00,231 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648745971 type: FLUSH version: 2 ttl: 0 2023-08-10T06:26:00,231 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system 2023-08-10T06:26:00,234 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:26:00,234 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,241 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/799f23d6b78042de88c6f772a308daa3 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/meta/799f23d6b78042de88c6f772a308daa3 2023-08-10T06:26:00,241 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:26:00,243 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/266080e572ea4022acd13bcdab5cae71 to hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/archive/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/266080e572ea4022acd13bcdab5cae71 2023-08-10T06:26:00,244 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:26:00,244 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648745971 type: FLUSH version: 2 ttl: 0 2023-08-10T06:26:00,245 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691648760244"}]},"ts":"9223372036854775807"} 2023-08-10T06:26:00,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:26:00,247 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:26:00,247 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691648760245"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691648760245"}]},"ts":"1691648760245"} 2023-08-10T06:26:00,248 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:26:00,248 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:26:00,248 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:26:00,248 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:26:00,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=72, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691648745971 type: FLUSH version: 2 ttl: 0 ) in 436 msec 2023-08-10T06:26:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=72 2023-08-10T06:26:00,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691648757685" 2023-08-10T06:26:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691648757685 2023-08-10T06:26:00,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:26:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] procedure2.ProcedureExecutor(1033): Stored pid=73, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:26:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-08-10T06:26:00,428 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:26:00,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648760428"}]},"ts":"1691648760428"} 2023-08-10T06:26:00,429 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:26:00,431 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:26:00,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN}] 2023-08-10T06:26:00,433 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN 2023-08-10T06:26:00,433 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:26:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-08-10T06:26:00,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:26:00,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124}] 2023-08-10T06:26:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-08-10T06:26:00,742 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(130): Open backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:00,742 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7254): Opening region: {ENCODED => bc059d26009e8ee1c786aed65c192ae9, NAME => 'backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:26:00,742 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:26:00,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(888): Instantiated backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:26:00,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7296): checking encryption for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7299): checking classloading for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,744 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,745 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName meta 2023-08-10T06:26:00,745 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:26:00,745 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:26:00,745 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,746 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc059d26009e8ee1c786aed65c192ae9 columnFamilyName session 2023-08-10T06:26:00,746 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:26:00,752 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:26:00,756 DEBUG [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/8d744d1dd5cd4cd29116171f0ff17678 2023-08-10T06:26:00,757 INFO [StoreOpener-bc059d26009e8ee1c786aed65c192ae9-1 {}] regionserver.HStore(324): Store=bc059d26009e8ee1c786aed65c192ae9/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:26:00,757 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:26:00,761 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1076): writing seq id for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:00,761 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): Opened bc059d26009e8ee1c786aed65c192ae9; next sequenceid=58; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9646764800, jitterRate=-0.10157501697540283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:26:00,762 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(992): Region open journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:26:00,763 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., pid=75, masterSystemTime=1691648760739 2023-08-10T06:26:00,764 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:00,764 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(158): Opened backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:00,764 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=bc059d26009e8ee1c786aed65c192ae9, regionState=OPEN, openSeqNum=58, regionLocation=jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:26:00,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=75, resume processing ppid=74 2023-08-10T06:26:00,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=75, ppid=74, state=SUCCESS; OpenRegionProcedure bc059d26009e8ee1c786aed65c192ae9, server=jenkins-hbase3.apache.org,44603,1691648705124 in 179 msec 2023-08-10T06:26:00,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=74, resume processing ppid=73 2023-08-10T06:26:00,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=74, ppid=73, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bc059d26009e8ee1c786aed65c192ae9, ASSIGN in 336 msec 2023-08-10T06:26:00,769 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691648760769"}]},"ts":"1691648760769"} 2023-08-10T06:26:00,770 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:26:00,772 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:26:00,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=73, state=SUCCESS; EnableTableProcedure table=backup:system in 349 msec 2023-08-10T06:26:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=73 2023-08-10T06:26:01,031 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:26:01,031 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:26:01,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:26:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:26:01,044 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:26:01,044 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-08-10T06:26:01,046 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] impl.BackupSystemTable(637): Finish backup exclusive operation REPAIR status: finished repair failed session: backup_1691648745954 2023-08-10T06:26:01,052 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:01,052 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:01,122 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.ResourceChecker(175): after: backup.TestBackupRepair#testFullBackupWithFailuresAndRestore Thread=604 (was 583) Potentially hanging thread: hconnection-0x7eccc166-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,44603,1691648705124' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55613@0x4044a6bc-SendThread(127.0.0.1:55613) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1926036644_16 at /127.0.0.1:53930 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x10b3a86c-shared-pool-27 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55613@0x45b2201b-SendThread(127.0.0.1:55613) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1247871637_16 at /127.0.0.1:34034 [Receiving block BP-2022367967-172.31.12.81-1691648698437:blk_1073741902_1078] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55613@0x45b2201b java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$488/570607460.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7eccc166-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,43813,1691648701870)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x10b3a86c-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1926036644_16 at /127.0.0.1:53926 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55613@0x4044a6bc-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1926036644_16 at /127.0.0.1:53864 [Waiting for operation #30] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:44603-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x10b3a86c-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x10b3a86c-shared-pool-26 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1247871637_16 at /127.0.0.1:34020 [Receiving block BP-2022367967-172.31.12.81-1691648698437:blk_1073741901_1077] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55613@0x4044a6bc java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$488/570607460.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7eccc166-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55613@0x45b2201b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x10b3a86c-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44603,1691648705124)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7eccc166-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2022367967-172.31.12.81-1691648698437:blk_1073741901_1077, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x10b3a86c-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2022367967-172.31.12.81-1691648698437:blk_1073741902_1078, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7eccc166-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=681 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=137 (was 135) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 171), AvailableMemoryMB=1847 (was 2174) 2023-08-10T06:26:01,123 WARN [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-08-10T06:26:01,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648748275_default_test-1691648713558" 2023-08-10T06:26:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648748275_default_test-1691648713558 2023-08-10T06:26:01,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691648752394_ns2_test-16916487135581" 2023-08-10T06:26:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43813 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691648752394_ns2_test-16916487135581 2023-08-10T06:26:01,141 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-08-10T06:26:01,141 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:01,142 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-08-10T06:26:01,142 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] util.JVMClusterUtil(257): Found active master hash=1602685828, stopped=false 2023-08-10T06:26:01,142 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:26:01,146 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] procedure2.ProcedureExecutor(631): Stopping 2023-08-10T06:26:01,147 DEBUG [Listener at jenkins-hbase3.apache.org/44547 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:01,147 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,44603,1691648705124' ***** 2023-08-10T06:26:01,147 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-08-10T06:26:01,147 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:26:01,161 INFO [RS:0;jenkins-hbase3:44603 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@4969a0f2{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:26:01,165 INFO [RS:0;jenkins-hbase3:44603 {}] server.AbstractConnector(383): Stopped ServerConnector@7207012b{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:26:01,166 INFO [RS:0;jenkins-hbase3:44603 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:26:01,166 INFO [RS:0;jenkins-hbase3:44603 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@11566dea{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:26:01,167 INFO [RS:0;jenkins-hbase3:44603 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@16e118f3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.log.dir/,STOPPED} 2023-08-10T06:26:01,169 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HeapMemoryManager(220): Stopping 2023-08-10T06:26:01,169 INFO [RS:0;jenkins-hbase3:44603 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-08-10T06:26:01,169 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-08-10T06:26:01,169 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for a9ff049a133a7be329fe9b42f1efbce8 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for 13ebfcea9d0422eea8430a5d6dd6cb62 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for 7643774d75fc83e3da00831f55e79730 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for 880494b3d2545260caf6635e12cd4de8 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for 81197e323905f3bf93978d985f6c1d3d 2023-08-10T06:26:01,170 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:01,171 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for 768720512dfecc14f1d87b27a2e78667 2023-08-10T06:26:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a9ff049a133a7be329fe9b42f1efbce8, disabling compactions & flushes 2023-08-10T06:26:01,171 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,44603,1691648705124 2023-08-10T06:26:01,171 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:26:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:26:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. after waiting 0 ms 2023-08-10T06:26:01,171 DEBUG [RS:0;jenkins-hbase3:44603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:01,171 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:26:01,171 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-08-10T06:26:01,171 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-08-10T06:26:01,171 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-08-10T06:26:01,172 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-08-10T06:26:01,172 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-08-10T06:26:01,172 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1583): Online Regions={a9ff049a133a7be329fe9b42f1efbce8=ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8., 13ebfcea9d0422eea8430a5d6dd6cb62=hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62., 7643774d75fc83e3da00831f55e79730=test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730., 880494b3d2545260caf6635e12cd4de8=ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8., 1588230740=hbase:meta,,1.1588230740, 81197e323905f3bf93978d985f6c1d3d=ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d., bc059d26009e8ee1c786aed65c192ae9=backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9., 768720512dfecc14f1d87b27a2e78667=backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667.} 2023-08-10T06:26:01,172 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:26:01,173 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:26:01,173 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:26:01,173 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:26:01,173 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:26:01,173 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=17.20 KB heapSize=30.02 KB 2023-08-10T06:26:01,173 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1609): Waiting on 13ebfcea9d0422eea8430a5d6dd6cb62, 1588230740, 7643774d75fc83e3da00831f55e79730, 768720512dfecc14f1d87b27a2e78667, 81197e323905f3bf93978d985f6c1d3d, 880494b3d2545260caf6635e12cd4de8, a9ff049a133a7be329fe9b42f1efbce8, bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:01,177 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns3/test-16916487135582/a9ff049a133a7be329fe9b42f1efbce8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:26:01,178 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:01,179 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:26:01,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a9ff049a133a7be329fe9b42f1efbce8: 2023-08-10T06:26:01,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16916487135582,,1691648717475.a9ff049a133a7be329fe9b42f1efbce8. 2023-08-10T06:26:01,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 13ebfcea9d0422eea8430a5d6dd6cb62, disabling compactions & flushes 2023-08-10T06:26:01,180 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:26:01,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:26:01,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. after waiting 0 ms 2023-08-10T06:26:01,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:26:01,180 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 13ebfcea9d0422eea8430a5d6dd6cb62 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-08-10T06:26:01,189 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:26:01,200 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/.tmp/info/c70d21780e24489dae8828fe5e8d719f is 45, key is default/info:d/1691648708475/Put/seqid=0 2023-08-10T06:26:01,202 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/.tmp/info/014b0b6d29a3429791f4899403fd081a is 159, key is ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8./info:regioninfo/1691648716463/Put/seqid=0 2023-08-10T06:26:01,373 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1609): Waiting on 13ebfcea9d0422eea8430a5d6dd6cb62, 1588230740, 7643774d75fc83e3da00831f55e79730, 768720512dfecc14f1d87b27a2e78667, 81197e323905f3bf93978d985f6c1d3d, 880494b3d2545260caf6635e12cd4de8, bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:01,574 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1609): Waiting on 13ebfcea9d0422eea8430a5d6dd6cb62, 1588230740, 7643774d75fc83e3da00831f55e79730, 768720512dfecc14f1d87b27a2e78667, 81197e323905f3bf93978d985f6c1d3d, 880494b3d2545260caf6635e12cd4de8, bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:01,606 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/.tmp/info/c70d21780e24489dae8828fe5e8d719f 2023-08-10T06:26:01,612 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c70d21780e24489dae8828fe5e8d719f 2023-08-10T06:26:01,612 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/.tmp/info/c70d21780e24489dae8828fe5e8d719f as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/info/c70d21780e24489dae8828fe5e8d719f 2023-08-10T06:26:01,614 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.80 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/.tmp/info/014b0b6d29a3429791f4899403fd081a 2023-08-10T06:26:01,618 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/info/c70d21780e24489dae8828fe5e8d719f, entries=7, sequenceid=11, filesize=5.1 K 2023-08-10T06:26:01,619 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 014b0b6d29a3429791f4899403fd081a 2023-08-10T06:26:01,619 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 13ebfcea9d0422eea8430a5d6dd6cb62 in 439ms, sequenceid=11, compaction requested=false 2023-08-10T06:26:01,622 DEBUG [StoreFileCloser-13ebfcea9d0422eea8430a5d6dd6cb62-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: c70d21780e24489dae8828fe5e8d719f 2023-08-10T06:26:01,626 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/namespace/13ebfcea9d0422eea8430a5d6dd6cb62/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-08-10T06:26:01,627 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:01,627 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:26:01,627 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 13ebfcea9d0422eea8430a5d6dd6cb62: 2023-08-10T06:26:01,627 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1691648707876.13ebfcea9d0422eea8430a5d6dd6cb62. 2023-08-10T06:26:01,628 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7643774d75fc83e3da00831f55e79730, disabling compactions & flushes 2023-08-10T06:26:01,628 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:26:01,628 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:26:01,628 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. after waiting 0 ms 2023-08-10T06:26:01,628 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:26:01,630 DEBUG [StoreFileCloser-7643774d75fc83e3da00831f55e79730-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e48b4016b9274b1b97275ba5bc817371 2023-08-10T06:26:01,637 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/default/test-1691648713558/7643774d75fc83e3da00831f55e79730/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-08-10T06:26:01,637 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:01,638 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:26:01,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7643774d75fc83e3da00831f55e79730: 2023-08-10T06:26:01,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1691648713558,,1691648714042.7643774d75fc83e3da00831f55e79730. 2023-08-10T06:26:01,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 880494b3d2545260caf6635e12cd4de8, disabling compactions & flushes 2023-08-10T06:26:01,638 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:26:01,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:26:01,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. after waiting 0 ms 2023-08-10T06:26:01,638 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:26:01,639 DEBUG [StoreFileCloser-880494b3d2545260caf6635e12cd4de8-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 46b9360ac0ee400d972892288374a1c2 2023-08-10T06:26:01,643 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns2/test-16916487135581/880494b3d2545260caf6635e12cd4de8/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-08-10T06:26:01,643 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/.tmp/table/7d05760863c24616ba24e24f96447ca9 is 59, key is ns2:test-16916487135581/table:state/1691648716473/Put/seqid=0 2023-08-10T06:26:01,644 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:01,645 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:26:01,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 880494b3d2545260caf6635e12cd4de8: 2023-08-10T06:26:01,645 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16916487135581,,1691648715276.880494b3d2545260caf6635e12cd4de8. 2023-08-10T06:26:01,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 81197e323905f3bf93978d985f6c1d3d, disabling compactions & flushes 2023-08-10T06:26:01,646 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:26:01,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:26:01,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. after waiting 0 ms 2023-08-10T06:26:01,646 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:26:01,658 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/ns4/test-16916487135583/81197e323905f3bf93978d985f6c1d3d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:26:01,659 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:01,659 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:26:01,659 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 81197e323905f3bf93978d985f6c1d3d: 2023-08-10T06:26:01,660 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16916487135583,,1691648719601.81197e323905f3bf93978d985f6c1d3d. 2023-08-10T06:26:01,660 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing bc059d26009e8ee1c786aed65c192ae9, disabling compactions & flushes 2023-08-10T06:26:01,660 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:01,660 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:01,660 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. after waiting 0 ms 2023-08-10T06:26:01,660 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:01,660 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing bc059d26009e8ee1c786aed65c192ae9 2/2 column families, dataSize=324 B heapSize=984 B 2023-08-10T06:26:01,661 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/.tmp/table/7d05760863c24616ba24e24f96447ca9 2023-08-10T06:26:01,666 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7d05760863c24616ba24e24f96447ca9 2023-08-10T06:26:01,667 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/.tmp/info/014b0b6d29a3429791f4899403fd081a as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/info/014b0b6d29a3429791f4899403fd081a 2023-08-10T06:26:01,674 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/info/014b0b6d29a3429791f4899403fd081a, entries=74, sequenceid=79, filesize=14.4 K 2023-08-10T06:26:01,675 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/.tmp/table/7d05760863c24616ba24e24f96447ca9 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/table/7d05760863c24616ba24e24f96447ca9 2023-08-10T06:26:01,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/df48a850e3ab4c1cbfd5a52d51782554 is 284, key is session:backup_1691648745954/session:context/1691648761045/Put/seqid=0 2023-08-10T06:26:01,681 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/table/7d05760863c24616ba24e24f96447ca9, entries=15, sequenceid=79, filesize=5.8 K 2023-08-10T06:26:01,682 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~17.20 KB/17610, heapSize ~29.73 KB/30448, currentSize=0 B/0 for 1588230740 in 509ms, sequenceid=79, compaction requested=false 2023-08-10T06:26:01,687 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 014b0b6d29a3429791f4899403fd081a 2023-08-10T06:26:01,690 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7d05760863c24616ba24e24f96447ca9 2023-08-10T06:26:01,694 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=1 2023-08-10T06:26:01,695 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:01,695 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:26:01,695 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:26:01,695 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:26:01,696 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-08-10T06:26:01,774 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1609): Waiting on 768720512dfecc14f1d87b27a2e78667, bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:01,863 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-08-10T06:26:01,863 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-08-10T06:26:01,974 DEBUG [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1609): Waiting on 768720512dfecc14f1d87b27a2e78667, bc059d26009e8ee1c786aed65c192ae9 2023-08-10T06:26:02,034 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691648713558 2023-08-10T06:26:02,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916487135583 2023-08-10T06:26:02,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:26:02,035 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:26:02,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916487135581 2023-08-10T06:26:02,036 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:26:02,037 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916487135582 2023-08-10T06:26:02,090 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/df48a850e3ab4c1cbfd5a52d51782554 2023-08-10T06:26:02,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: df48a850e3ab4c1cbfd5a52d51782554 2023-08-10T06:26:02,096 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/.tmp/session/df48a850e3ab4c1cbfd5a52d51782554 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/df48a850e3ab4c1cbfd5a52d51782554 2023-08-10T06:26:02,103 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/session/df48a850e3ab4c1cbfd5a52d51782554, entries=2, sequenceid=62, filesize=5.2 K 2023-08-10T06:26:02,103 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~712 B/712, currentSize=0 B/0 for bc059d26009e8ee1c786aed65c192ae9 in 443ms, sequenceid=62, compaction requested=true 2023-08-10T06:26:02,104 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 8d744d1dd5cd4cd29116171f0ff17678 2023-08-10T06:26:02,104 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 836983030dee4b4c92ecdbbb02dcd61c 2023-08-10T06:26:02,105 DEBUG [StoreFileCloser-bc059d26009e8ee1c786aed65c192ae9-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: df48a850e3ab4c1cbfd5a52d51782554 2023-08-10T06:26:02,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system/bc059d26009e8ee1c786aed65c192ae9/recovered.edits/65.seqid, newMaxSeqId=65, maxSeqId=57 2023-08-10T06:26:02,110 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:02,111 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:02,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for bc059d26009e8ee1c786aed65c192ae9: 2023-08-10T06:26:02,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1691648720901.bc059d26009e8ee1c786aed65c192ae9. 2023-08-10T06:26:02,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 768720512dfecc14f1d87b27a2e78667, disabling compactions & flushes 2023-08-10T06:26:02,111 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:26:02,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:26:02,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. after waiting 0 ms 2023-08-10T06:26:02,111 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:26:02,114 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/data/backup/system_bulk/768720512dfecc14f1d87b27a2e78667/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:26:02,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:26:02,115 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:26:02,115 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 768720512dfecc14f1d87b27a2e78667: 2023-08-10T06:26:02,116 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1691648722034.768720512dfecc14f1d87b27a2e78667. 2023-08-10T06:26:02,174 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,44603,1691648705124; all regions closed. 2023-08-10T06:26:02,186 DEBUG [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/oldWALs 2023-08-10T06:26:02,186 INFO [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44603%2C1691648705124.meta:.meta(num 1691648748127) 2023-08-10T06:26:02,194 DEBUG [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/oldWALs 2023-08-10T06:26:02,194 INFO [RS:0;jenkins-hbase3:44603 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44603%2C1691648705124:(num 1691648748109) 2023-08-10T06:26:02,194 DEBUG [RS:0;jenkins-hbase3:44603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:02,194 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:26:02,194 INFO [RS:0;jenkins-hbase3:44603 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-08-10T06:26:02,195 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:26:02,196 INFO [RS:0;jenkins-hbase3:44603 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:44603 2023-08-10T06:26:02,205 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,44603,1691648705124] 2023-08-10T06:26:02,205 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,44603,1691648705124; numProcessing=1 2023-08-10T06:26:02,207 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,44603,1691648705124 expired; onlineServers=0 2023-08-10T06:26:02,207 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,43813,1691648701870' ***** 2023-08-10T06:26:02,207 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-08-10T06:26:02,208 DEBUG [M:0;jenkins-hbase3:43813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1edaae39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:26:02,208 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:26:02,252 INFO [M:0;jenkins-hbase3:43813 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@42679f83{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:26:02,252 INFO [M:0;jenkins-hbase3:43813 {}] server.AbstractConnector(383): Stopped ServerConnector@57e8e0d7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:26:02,252 INFO [M:0;jenkins-hbase3:43813 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:26:02,253 INFO [M:0;jenkins-hbase3:43813 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@290f83bc{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:26:02,254 INFO [M:0;jenkins-hbase3:43813 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@488cf936{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/hadoop.log.dir/,STOPPED} 2023-08-10T06:26:02,254 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43813,1691648701870 2023-08-10T06:26:02,254 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43813,1691648701870; all regions closed. 2023-08-10T06:26:02,254 DEBUG [M:0;jenkins-hbase3:43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:02,254 INFO [M:0;jenkins-hbase3:43813 {}] master.HMaster(1664): Stopping master jetty server 2023-08-10T06:26:02,255 INFO [M:0;jenkins-hbase3:43813 {}] server.AbstractConnector(383): Stopped ServerConnector@bfaf459{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:26:02,255 DEBUG [M:0;jenkins-hbase3:43813 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-08-10T06:26:02,255 INFO [M:0;jenkins-hbase3:43813 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-08-10T06:26:02,255 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-08-10T06:26:02,256 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648706611 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691648706611,5,FailOnTimeoutGroup] 2023-08-10T06:26:02,256 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648706614 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691648706614,5,FailOnTimeoutGroup] 2023-08-10T06:26:02,256 DEBUG [M:0;jenkins-hbase3:43813 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:26:02,256 DEBUG [M:0;jenkins-hbase3:43813 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-08-10T06:26:02,256 INFO [M:0;jenkins-hbase3:43813 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-08-10T06:26:02,256 DEBUG [M:0;jenkins-hbase3:43813 {}] master.HMaster(1687): Stopping service threads 2023-08-10T06:26:02,256 INFO [M:0;jenkins-hbase3:43813 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-08-10T06:26:02,256 ERROR [M:0;jenkins-hbase3:43813 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-08-10T06:26:02,257 INFO [M:0;jenkins-hbase3:43813 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-08-10T06:26:02,257 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-08-10T06:26:02,257 WARN [M:0;jenkins-hbase3:43813 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-08-10T06:26:02,258 INFO [M:0;jenkins-hbase3:43813 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-08-10T06:26:02,258 INFO [M:0;jenkins-hbase3:43813 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-08-10T06:26:02,258 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:26:02,269 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:26:02,269 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:26:02,270 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:26:02,270 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:26:02,270 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=285.11 KB heapSize=342.27 KB 2023-08-10T06:26:02,282 DEBUG [M:0;jenkins-hbase3:43813 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3042654bcb9547f389ef15657812ef42 is 88, key is hbase:meta,,1/info:sn/1691648707224/Put/seqid=0 2023-08-10T06:26:02,307 INFO [RS:0;jenkins-hbase3:44603 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,44603,1691648705124; zookeeper connection closed. 2023-08-10T06:26:02,307 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4d906a0 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4d906a0 2023-08-10T06:26:02,308 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-08-10T06:26:02,687 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3042654bcb9547f389ef15657812ef42 2023-08-10T06:26:02,692 DEBUG [M:0;jenkins-hbase3:43813 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3042654bcb9547f389ef15657812ef42 2023-08-10T06:26:02,706 DEBUG [M:0;jenkins-hbase3:43813 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4fb066c6548349b3b4bab7f43cc078b4 is 1037, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1691648731158/Put/seqid=0 2023-08-10T06:26:03,111 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=284.52 KB at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4fb066c6548349b3b4bab7f43cc078b4 2023-08-10T06:26:03,117 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4fb066c6548349b3b4bab7f43cc078b4 2023-08-10T06:26:03,117 DEBUG [M:0;jenkins-hbase3:43813 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4fb066c6548349b3b4bab7f43cc078b4 2023-08-10T06:26:03,131 DEBUG [M:0;jenkins-hbase3:43813 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ac47c84a328c4680bf132b171798b019 is 82, key is jenkins-hbase3.apache.org,44603,1691648705124/rs:state/1691648706745/Put/seqid=0 2023-08-10T06:26:03,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:26:03,536 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=679 (bloomFilter=true), to=hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ac47c84a328c4680bf132b171798b019 2023-08-10T06:26:03,542 DEBUG [M:0;jenkins-hbase3:43813 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ac47c84a328c4680bf132b171798b019 2023-08-10T06:26:03,543 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3042654bcb9547f389ef15657812ef42 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3042654bcb9547f389ef15657812ef42 2023-08-10T06:26:03,549 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3042654bcb9547f389ef15657812ef42, entries=8, sequenceid=679, filesize=5.5 K 2023-08-10T06:26:03,550 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4fb066c6548349b3b4bab7f43cc078b4 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4fb066c6548349b3b4bab7f43cc078b4 2023-08-10T06:26:03,556 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4fb066c6548349b3b4bab7f43cc078b4 2023-08-10T06:26:03,556 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4fb066c6548349b3b4bab7f43cc078b4, entries=75, sequenceid=679, filesize=23.3 K 2023-08-10T06:26:03,557 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ac47c84a328c4680bf132b171798b019 as hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ac47c84a328c4680bf132b171798b019 2023-08-10T06:26:03,563 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38181/user/jenkins/test-data/a4caa99d-7c26-4105-df0c-0270ee4d8ad0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ac47c84a328c4680bf132b171798b019, entries=1, sequenceid=679, filesize=5.1 K 2023-08-10T06:26:03,564 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(3022): Finished flush of dataSize ~285.11 KB/291955, heapSize ~341.97 KB/350176, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1294ms, sequenceid=679, compaction requested=false 2023-08-10T06:26:03,567 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3042654bcb9547f389ef15657812ef42 2023-08-10T06:26:03,574 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 4fb066c6548349b3b4bab7f43cc078b4 2023-08-10T06:26:03,575 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: ac47c84a328c4680bf132b171798b019 2023-08-10T06:26:03,576 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:26:03,576 DEBUG [M:0;jenkins-hbase3:43813 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:26:03,579 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:26:03,579 INFO [M:0;jenkins-hbase3:43813 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-08-10T06:26:03,579 INFO [M:0;jenkins-hbase3:43813 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-08-10T06:26:03,580 INFO [M:0;jenkins-hbase3:43813 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:43813 2023-08-10T06:26:03,686 INFO [M:0;jenkins-hbase3:43813 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43813,1691648701870; zookeeper connection closed. 2023-08-10T06:26:03,689 WARN [Listener at jenkins-hbase3.apache.org/44547 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-08-10T06:26:03,693 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:26:03,745 WARN [BP-2022367967-172.31.12.81-1691648698437 heartbeating to localhost/127.0.0.1:38181 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2022367967-172.31.12.81-1691648698437 (Datanode Uuid d4546392-d55d-4907-adb9-1e17c2760bef) service to localhost/127.0.0.1:38181 2023-08-10T06:26:03,747 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/dfs/data/data1/current/BP-2022367967-172.31.12.81-1691648698437 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:26:03,748 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c2af9d89-a05e-ddf6-da30-ecc8c250e653/cluster_44480888-31fd-3c60-a504-fc20aeb77658/dfs/data/data2/current/BP-2022367967-172.31.12.81-1691648698437 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:26:03,841 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:26:03,903 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-08-10T06:26:03,903 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-08-10T06:26:03,913 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:26:04,032 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:26:04,145 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:26:04,145 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:26:04,250 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-08-10T06:26:04,258 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-08-10T06:26:04,261 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:26:04,266 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:26:04,370 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:26:04,370 INFO [Listener at jenkins-hbase3.apache.org/44547 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped