2024-11-19 11:05:24,990 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-19 11:05:25,002 main DEBUG Took 0.010019 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 11:05:25,003 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 11:05:25,003 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 11:05:25,004 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 11:05:25,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,016 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 11:05:25,028 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,030 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,031 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,032 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,033 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,034 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,035 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,036 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,037 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,037 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,038 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,039 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,039 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 11:05:25,039 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,040 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 11:05:25,041 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 11:05:25,043 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 11:05:25,045 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 11:05:25,045 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 11:05:25,047 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 11:05:25,047 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 11:05:25,056 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 11:05:25,059 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 11:05:25,061 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 11:05:25,061 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 11:05:25,062 main DEBUG createAppenders(={Console}) 2024-11-19 11:05:25,063 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-19 11:05:25,063 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-19 11:05:25,063 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-19 11:05:25,064 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 11:05:25,064 main DEBUG OutputStream closed 2024-11-19 11:05:25,064 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 11:05:25,065 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 11:05:25,065 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-19 11:05:25,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 11:05:25,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 11:05:25,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 11:05:25,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 11:05:25,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 11:05:25,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 11:05:25,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 11:05:25,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 11:05:25,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 11:05:25,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 11:05:25,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 11:05:25,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 11:05:25,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 11:05:25,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 11:05:25,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 11:05:25,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 11:05:25,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 11:05:25,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 11:05:25,149 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 11:05:25,150 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-19 11:05:25,150 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 11:05:25,151 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-19T11:05:25,186 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-19 11:05:25,188 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 11:05:25,189 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T11:05:25,455 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe 2024-11-19T11:05:25,459 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-19T11:05:25,460 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:05:25,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T11:05:25,476 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566, deleteOnExit=true 2024-11-19T11:05:25,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T11:05:25,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/test.cache.data in system properties and HBase conf 2024-11-19T11:05:25,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T11:05:25,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir in system properties and HBase conf 2024-11-19T11:05:25,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T11:05:25,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T11:05:25,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T11:05:25,564 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T11:05:25,684 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T11:05:25,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T11:05:25,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T11:05:25,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T11:05:25,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T11:05:25,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T11:05:25,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T11:05:25,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T11:05:25,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T11:05:25,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T11:05:25,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/nfs.dump.dir in system properties and HBase conf 2024-11-19T11:05:25,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir in system properties and HBase conf 2024-11-19T11:05:25,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T11:05:25,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T11:05:25,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T11:05:26,990 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T11:05:27,079 INFO [Time-limited test {}] log.Log(170): Logging initialized @2818ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T11:05:27,171 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:27,242 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:05:27,271 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:27,271 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:27,273 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T11:05:27,291 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:27,294 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c623081{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:27,296 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4edc5bd1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T11:05:27,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4084a5e6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir/jetty-localhost-42069-hadoop-hdfs-3_4_1-tests_jar-_-any-5322989356828497862/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T11:05:27,511 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3151d644{HTTP/1.1, (http/1.1)}{localhost:42069} 2024-11-19T11:05:27,512 INFO [Time-limited test {}] server.Server(415): Started @3252ms 2024-11-19T11:05:28,096 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:28,106 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:05:28,108 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:28,109 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:28,109 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T11:05:28,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fdab1fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:28,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27593828{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T11:05:28,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@161bd8f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir/jetty-localhost-41549-hadoop-hdfs-3_4_1-tests_jar-_-any-9741261181181734839/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T11:05:28,214 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14a6ba7a{HTTP/1.1, (http/1.1)}{localhost:41549} 2024-11-19T11:05:28,214 INFO [Time-limited test {}] server.Server(415): Started @3954ms 2024-11-19T11:05:28,277 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T11:05:29,580 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/data/data2/current/BP-264477037-172.17.0.2-1732014326384/current, will proceed with Du for space computation calculation, 2024-11-19T11:05:29,580 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/data/data1/current/BP-264477037-172.17.0.2-1732014326384/current, will proceed with Du for space computation calculation, 2024-11-19T11:05:29,606 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T11:05:29,647 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd6d33628be12920 with lease ID 0x6c31ab2d579b4bb3: Processing first storage report for DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1 from datanode DatanodeRegistration(127.0.0.1:37071, datanodeUuid=2afb7002-8414-4297-b9f4-c050734cf990, infoPort=44235, infoSecurePort=0, ipcPort=37513, storageInfo=lv=-57;cid=testClusterID;nsid=2004040052;c=1732014326384) 2024-11-19T11:05:29,648 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd6d33628be12920 with lease ID 0x6c31ab2d579b4bb3: from storage DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1 node DatanodeRegistration(127.0.0.1:37071, datanodeUuid=2afb7002-8414-4297-b9f4-c050734cf990, infoPort=44235, infoSecurePort=0, ipcPort=37513, storageInfo=lv=-57;cid=testClusterID;nsid=2004040052;c=1732014326384), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T11:05:29,648 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcd6d33628be12920 with lease ID 0x6c31ab2d579b4bb3: Processing first storage report for DS-dc545a2b-6ccf-4a87-9a2c-b2a42a4b6e8e from datanode DatanodeRegistration(127.0.0.1:37071, datanodeUuid=2afb7002-8414-4297-b9f4-c050734cf990, infoPort=44235, infoSecurePort=0, ipcPort=37513, storageInfo=lv=-57;cid=testClusterID;nsid=2004040052;c=1732014326384) 2024-11-19T11:05:29,648 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcd6d33628be12920 with lease ID 0x6c31ab2d579b4bb3: from storage DS-dc545a2b-6ccf-4a87-9a2c-b2a42a4b6e8e node DatanodeRegistration(127.0.0.1:37071, datanodeUuid=2afb7002-8414-4297-b9f4-c050734cf990, infoPort=44235, infoSecurePort=0, ipcPort=37513, storageInfo=lv=-57;cid=testClusterID;nsid=2004040052;c=1732014326384), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T11:05:29,698 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe 2024-11-19T11:05:29,770 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/zookeeper_0, clientPort=51477, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T11:05:29,779 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51477 2024-11-19T11:05:29,790 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:29,793 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:29,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741825_1001 (size=7) 2024-11-19T11:05:30,396 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 with version=8 2024-11-19T11:05:30,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/hbase-staging 2024-11-19T11:05:30,408 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-19T11:05:30,409 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-19T11:05:30,409 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-19T11:05:30,409 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-19T11:05:30,479 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T11:05:30,859 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c28903dc2d16:0 server-side Connection retries=45 2024-11-19T11:05:30,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:05:30,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T11:05:30,872 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T11:05:30,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:05:30,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T11:05:31,001 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T11:05:31,063 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T11:05:31,073 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T11:05:31,076 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T11:05:31,102 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 135737 (auto-detected) 2024-11-19T11:05:31,103 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T11:05:31,122 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39427 2024-11-19T11:05:31,147 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39427 connecting to ZooKeeper ensemble=127.0.0.1:51477 2024-11-19T11:05:31,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394270x0, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T11:05:31,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39427-0x10152c54b450000 connected 2024-11-19T11:05:31,358 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:31,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:31,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-19T11:05:31,402 INFO [Time-limited test {}] log.Log(170): Logging initialized @7142ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T11:05:31,446 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-19T11:05:31,446 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-19T11:05:31,447 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-19T11:05:31,448 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-19T11:05:31,449 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-19T11:05:31,450 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-19T11:05:31,451 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-19T11:05:31,451 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-19T11:05:31,453 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-19T11:05:31,511 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 33497 2024-11-19T11:05:31,512 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-19T11:05:31,525 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-19T11:05:31,527 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2f4c8752{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:31,527 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-19T11:05:31,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5f79216d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-19T11:05:31,567 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-19T11:05:31,574 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:31,575 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:31,576 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T11:05:31,580 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-19T11:05:32,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@1190ad88{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-19T11:05:32,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@28c6e73f{HTTP/1.1, (http/1.1)}{0.0.0.0:33497} 2024-11-19T11:05:32,115 INFO [Time-limited test {}] server.Server(415): Started @7855ms 2024-11-19T11:05:32,118 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442, hbase.cluster.distributed=false 2024-11-19T11:05:32,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-19T11:05:32,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39427 2024-11-19T11:05:32,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39427 2024-11-19T11:05:32,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39427 2024-11-19T11:05:32,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39427 2024-11-19T11:05:32,146 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39427 2024-11-19T11:05:32,277 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c28903dc2d16:0 server-side Connection retries=45 2024-11-19T11:05:32,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:05:32,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T11:05:32,280 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T11:05:32,280 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T11:05:32,280 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T11:05:32,284 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T11:05:32,287 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T11:05:32,288 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43005 2024-11-19T11:05:32,290 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43005 connecting to ZooKeeper ensemble=127.0.0.1:51477 2024-11-19T11:05:32,291 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:32,294 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:32,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430050x0, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T11:05:32,347 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430050x0, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-19T11:05:32,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43005-0x10152c54b450001 connected 2024-11-19T11:05:32,349 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-19T11:05:32,350 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-19T11:05:32,350 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-19T11:05:32,350 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-19T11:05:32,351 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-19T11:05:32,352 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-19T11:05:32,352 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-19T11:05:32,353 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-19T11:05:32,353 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-19T11:05:32,355 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 32959 2024-11-19T11:05:32,356 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-19T11:05:32,358 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-19T11:05:32,358 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@340a8627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:32,358 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-19T11:05:32,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3b20ef39{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-19T11:05:32,372 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-19T11:05:32,373 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:32,374 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:32,374 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T11:05:32,375 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-19T11:05:32,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@7cd2e4d8{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-19T11:05:32,379 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@91e124c{HTTP/1.1, (http/1.1)}{0.0.0.0:32959} 2024-11-19T11:05:32,379 INFO [Time-limited test {}] server.Server(415): Started @8119ms 2024-11-19T11:05:32,383 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T11:05:32,390 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T11:05:32,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-19T11:05:32,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-19T11:05:32,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43005 2024-11-19T11:05:32,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43005 2024-11-19T11:05:32,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43005 2024-11-19T11:05:32,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43005 2024-11-19T11:05:32,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43005 2024-11-19T11:05:32,414 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c28903dc2d16:39427 2024-11-19T11:05:32,417 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-19T11:05:32,421 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@39c3cce8{HTTP/1.1, (http/1.1)}{0.0.0.0:45881} 2024-11-19T11:05:32,422 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] server.Server(415): Started @8162ms 2024-11-19T11:05:32,422 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/c28903dc2d16,39427,1732014330695 2024-11-19T11:05:32,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-19T11:05:32,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-19T11:05:32,432 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/c28903dc2d16,39427,1732014330695 2024-11-19T11:05:32,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-19T11:05:32,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:32,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:32,462 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-19T11:05:32,463 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/c28903dc2d16,39427,1732014330695 from backup master directory 2024-11-19T11:05:32,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/c28903dc2d16,39427,1732014330695 2024-11-19T11:05:32,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-19T11:05:32,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-19T11:05:32,473 WARN [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T11:05:32,473 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c28903dc2d16,39427,1732014330695 2024-11-19T11:05:32,476 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T11:05:32,477 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T11:05:32,530 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/hbase.id] with ID: 1bc60509-da67-44f5-84a2-d7781db41c37 2024-11-19T11:05:32,530 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.tmp/hbase.id 2024-11-19T11:05:32,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741826_1002 (size=42) 2024-11-19T11:05:32,943 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.tmp/hbase.id]:[hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/hbase.id] 2024-11-19T11:05:32,993 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T11:05:32,999 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T11:05:33,024 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-19T11:05:33,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:33,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:33,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741827_1003 (size=196) 2024-11-19T11:05:33,504 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:33,506 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T11:05:33,513 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-19T11:05:33,516 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-19T11:05:33,527 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:05:33,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741828_1004 (size=1189) 2024-11-19T11:05:33,983 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store 2024-11-19T11:05:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741829_1005 (size=34) 2024-11-19T11:05:34,410 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T11:05:34,413 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:34,415 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T11:05:34,415 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:05:34,416 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:05:34,417 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T11:05:34,418 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:05:34,418 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:05:34,420 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732014334415Disabling compacts and flushes for region at 1732014334415Disabling writes for close at 1732014334418 (+3 ms)Writing region close event to WAL at 1732014334418Closed at 1732014334418 2024-11-19T11:05:34,422 WARN [master/c28903dc2d16:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/.initializing 2024-11-19T11:05:34,423 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/WALs/c28903dc2d16,39427,1732014330695 2024-11-19T11:05:34,429 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:05:34,435 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c28903dc2d16,39427,1732014330695.regiongroup-0 2024-11-19T11:05:34,453 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28903dc2d16%2C39427%2C1732014330695.c28903dc2d16%2C39427%2C1732014330695.regiongroup-0, suffix=, logDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/WALs/c28903dc2d16,39427,1732014330695, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/oldWALs, maxLogs=10 2024-11-19T11:05:34,487 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/WALs/c28903dc2d16,39427,1732014330695/c28903dc2d16%2C39427%2C1732014330695.c28903dc2d16%2C39427%2C1732014330695.regiongroup-0.1732014334460, exclude list is [], retry=0 2024-11-19T11:05:34,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:34,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T11:05:34,543 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/WALs/c28903dc2d16,39427,1732014330695/c28903dc2d16%2C39427%2C1732014330695.c28903dc2d16%2C39427%2C1732014330695.regiongroup-0.1732014334460 2024-11-19T11:05:34,544 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:34,545 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:34,545 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:34,548 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,549 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,612 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T11:05:34,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:34,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:34,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T11:05:34,622 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:34,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:34,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T11:05:34,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:34,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:34,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T11:05:34,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:34,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:34,633 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,637 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,639 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,646 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,647 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,651 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T11:05:34,656 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T11:05:34,662 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:34,664 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9422428160, jitterRate=-0.1224679946899414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T11:05:34,675 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732014334564Initializing all the Stores at 1732014334566 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014334567 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014334568 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014334569 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014334569Cleaning up temporary data from old regions at 1732014334647 (+78 ms)Region opened successfully at 1732014334675 (+28 ms) 2024-11-19T11:05:34,677 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T11:05:34,713 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f748ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=c28903dc2d16/172.17.0.2:0 2024-11-19T11:05:34,747 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T11:05:34,760 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T11:05:34,760 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T11:05:34,764 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T11:05:34,765 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T11:05:34,771 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T11:05:34,771 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T11:05:34,802 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T11:05:34,812 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-19T11:05:34,861 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-19T11:05:34,864 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T11:05:34,866 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-19T11:05:34,871 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-19T11:05:34,874 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T11:05:34,879 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-19T11:05:34,892 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-19T11:05:34,894 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-19T11:05:34,903 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-19T11:05:34,940 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T11:05:34,945 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-19T11:05:34,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-19T11:05:34,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-19T11:05:34,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:34,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:34,959 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c28903dc2d16,39427,1732014330695, sessionid=0x10152c54b450000, setting cluster-up flag (Was=false) 2024-11-19T11:05:34,968 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-19T11:05:34,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:34,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:35,019 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-19T11:05:35,021 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c28903dc2d16,39427,1732014330695 2024-11-19T11:05:35,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:35,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:35,071 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-19T11:05:35,073 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c28903dc2d16,39427,1732014330695 2024-11-19T11:05:35,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:35,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:35,124 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-19T11:05:35,126 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c28903dc2d16,39427,1732014330695 2024-11-19T11:05:35,131 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T11:05:35,198 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T11:05:35,206 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(746): ClusterId : 1bc60509-da67-44f5-84a2-d7781db41c37 2024-11-19T11:05:35,207 INFO [RS:0;c28903dc2d16:43005 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-19T11:05:35,207 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T11:05:35,209 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T11:05:35,214 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T11:05:35,221 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T11:05:35,221 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-19T11:05:35,220 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c28903dc2d16,39427,1732014330695 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T11:05:35,231 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-19T11:05:35,231 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T11:05:35,232 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c28903dc2d16:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:05:35,232 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c28903dc2d16:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:05:35,232 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c28903dc2d16:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:05:35,232 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c28903dc2d16:0, corePoolSize=5, maxPoolSize=5 2024-11-19T11:05:35,233 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c28903dc2d16:0, corePoolSize=10, maxPoolSize=10 2024-11-19T11:05:35,233 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,233 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c28903dc2d16:0, corePoolSize=2, maxPoolSize=2 2024-11-19T11:05:35,233 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,234 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732014365234 2024-11-19T11:05:35,236 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T11:05:35,237 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T11:05:35,239 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T11:05:35,239 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T11:05:35,241 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T11:05:35,242 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T11:05:35,242 DEBUG [RS:0;c28903dc2d16:43005 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fed13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=c28903dc2d16/172.17.0.2:0 2024-11-19T11:05:35,242 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-19T11:05:35,242 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T11:05:35,243 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T11:05:35,243 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T11:05:35,244 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,246 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:35,246 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T11:05:35,247 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T11:05:35,248 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T11:05:35,249 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T11:05:35,258 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c28903dc2d16:43005 2024-11-19T11:05:35,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741831_1007 (size=1321) 2024-11-19T11:05:35,260 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494a1e75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:35,261 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:05:35,261 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T11:05:35,261 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T11:05:35,261 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T11:05:35,264 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(2659): reportForDuty to master=c28903dc2d16,39427,1732014330695 with port=43005, startcode=1732014332227 2024-11-19T11:05:35,273 DEBUG [RS:0;c28903dc2d16:43005 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T11:05:35,273 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:05:35,298 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:05:35,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T11:05:35,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:05:35,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:05:35,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea19eb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:35,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:05:35,314 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39427 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:05:35,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:05:35,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:35,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:05:35,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cae267a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:35,324 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-19T11:05:35,327 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T11:05:35,327 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T11:05:35,329 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c28903dc2d16:0:becomeActiveMaster-HFileCleaner.large.0-1732014335328,5,FailOnTimeoutGroup] 2024-11-19T11:05:35,330 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c28903dc2d16:0:becomeActiveMaster-HFileCleaner.small.0-1732014335329,5,FailOnTimeoutGroup] 2024-11-19T11:05:35,330 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,330 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T11:05:35,331 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,332 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,338 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-19T11:05:35,338 WARN [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-19T11:05:35,440 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(2659): reportForDuty to master=c28903dc2d16,39427,1732014330695 with port=43005, startcode=1732014332227 2024-11-19T11:05:35,442 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39427 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c28903dc2d16,43005,1732014332227 2024-11-19T11:05:35,444 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39427 {}] master.ServerManager(517): Registering regionserver=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:35,452 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:35,453 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35831 2024-11-19T11:05:35,453 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=33497 2024-11-19T11:05:35,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-19T11:05:35,498 DEBUG [RS:0;c28903dc2d16:43005 {}] zookeeper.ZKUtil(111): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rs/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:35,499 WARN [RS:0;c28903dc2d16:43005 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T11:05:35,499 INFO [RS:0;c28903dc2d16:43005 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-19T11:05:35,499 INFO [RS:0;c28903dc2d16:43005 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-19T11:05:35,499 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:35,510 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c28903dc2d16,43005,1732014332227] 2024-11-19T11:05:35,525 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T11:05:35,540 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T11:05:35,546 INFO [RS:0;c28903dc2d16:43005 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T11:05:35,546 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,549 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T11:05:35,555 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T11:05:35,556 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,556 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c28903dc2d16:0, corePoolSize=2, maxPoolSize=2 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,557 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,558 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,558 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,558 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c28903dc2d16:0, corePoolSize=1, maxPoolSize=1 2024-11-19T11:05:35,558 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0, corePoolSize=3, maxPoolSize=3 2024-11-19T11:05:35,558 DEBUG [RS:0;c28903dc2d16:43005 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c28903dc2d16:0, corePoolSize=3, maxPoolSize=3 2024-11-19T11:05:35,559 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,559 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,559 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,559 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,559 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,560 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,43005,1732014332227-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T11:05:35,578 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T11:05:35,580 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,43005,1732014332227-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,580 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,580 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.Replication(171): c28903dc2d16,43005,1732014332227 started 2024-11-19T11:05:35,596 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:35,596 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1482): Serving as c28903dc2d16,43005,1732014332227, RpcServer on c28903dc2d16/172.17.0.2:43005, sessionid=0x10152c54b450001 2024-11-19T11:05:35,597 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T11:05:35,597 DEBUG [RS:0;c28903dc2d16:43005 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c28903dc2d16,43005,1732014332227 2024-11-19T11:05:35,597 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c28903dc2d16,43005,1732014332227' 2024-11-19T11:05:35,597 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-19T11:05:35,598 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-19T11:05:35,599 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T11:05:35,599 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-19T11:05:35,599 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c28903dc2d16,43005,1732014332227' 2024-11-19T11:05:35,599 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:05:35,600 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:05:35,600 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-19T11:05:35,600 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-19T11:05:35,601 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T11:05:35,601 DEBUG [RS:0;c28903dc2d16:43005 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c28903dc2d16,43005,1732014332227 2024-11-19T11:05:35,601 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c28903dc2d16,43005,1732014332227' 2024-11-19T11:05:35,601 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-19T11:05:35,601 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-19T11:05:35,602 DEBUG [RS:0;c28903dc2d16:43005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T11:05:35,602 INFO [RS:0;c28903dc2d16:43005 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T11:05:35,602 INFO [RS:0;c28903dc2d16:43005 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T11:05:35,661 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T11:05:35,662 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:35,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741832_1008 (size=32) 2024-11-19T11:05:35,707 INFO [RS:0;c28903dc2d16:43005 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:05:35,707 INFO [RS:0;c28903dc2d16:43005 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c28903dc2d16,43005,1732014332227.regiongroup-0 2024-11-19T11:05:35,710 INFO [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0, suffix=, logDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs, maxLogs=32 2024-11-19T11:05:35,726 DEBUG [RS:0;c28903dc2d16:43005 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713, exclude list is [], retry=0 2024-11-19T11:05:35,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:35,735 INFO [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713 2024-11-19T11:05:35,736 DEBUG [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:36,075 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:36,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T11:05:36,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T11:05:36,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T11:05:36,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T11:05:36,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T11:05:36,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T11:05:36,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T11:05:36,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T11:05:36,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T11:05:36,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740 2024-11-19T11:05:36,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740 2024-11-19T11:05:36,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T11:05:36,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T11:05:36,101 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T11:05:36,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T11:05:36,107 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:36,108 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10996122240, jitterRate=0.024093687534332275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T11:05:36,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732014336076Initializing all the Stores at 1732014336078 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014336078Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014336078Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014336078Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014336078Cleaning up temporary data from old regions at 1732014336100 (+22 ms)Region opened successfully at 1732014336111 (+11 ms) 2024-11-19T11:05:36,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T11:05:36,111 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T11:05:36,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T11:05:36,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T11:05:36,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T11:05:36,113 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T11:05:36,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732014336111Disabling compacts and flushes for region at 1732014336111Disabling writes for close at 1732014336111Writing region close event to WAL at 1732014336113 (+2 ms)Closed at 1732014336113 2024-11-19T11:05:36,117 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T11:05:36,117 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T11:05:36,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T11:05:36,129 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T11:05:36,132 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T11:05:36,285 DEBUG [c28903dc2d16:39427 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T11:05:36,298 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:36,304 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c28903dc2d16,43005,1732014332227, state=OPENING 2024-11-19T11:05:36,366 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T11:05:36,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:36,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:05:36,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-19T11:05:36,378 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-19T11:05:36,379 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T11:05:36,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:36,556 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T11:05:36,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52681, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T11:05:36,570 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T11:05:36,570 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-19T11:05:36,571 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-19T11:05:36,571 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:05:36,571 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T11:05:36,575 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28903dc2d16%2C43005%2C1732014332227.meta, suffix=.meta, logDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs, maxLogs=32 2024-11-19T11:05:36,597 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014336578.meta, exclude list is [], retry=0 2024-11-19T11:05:36,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:36,604 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014336578.meta 2024-11-19T11:05:36,605 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:36,605 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:36,610 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:36,611 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T11:05:36,614 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T11:05:36,614 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T11:05:36,618 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T11:05:36,618 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:36,619 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T11:05:36,619 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T11:05:36,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T11:05:36,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T11:05:36,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T11:05:36,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T11:05:36,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T11:05:36,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T11:05:36,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T11:05:36,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T11:05:36,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:36,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T11:05:36,632 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T11:05:36,633 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740 2024-11-19T11:05:36,636 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740 2024-11-19T11:05:36,638 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T11:05:36,638 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T11:05:36,639 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T11:05:36,642 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T11:05:36,643 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9797081440, jitterRate=-0.08757568895816803}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T11:05:36,643 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T11:05:36,644 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732014336620Writing region info on filesystem at 1732014336620Initializing all the Stores at 1732014336621 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014336621Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014336622 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014336622Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732014336622Cleaning up temporary data from old regions at 1732014336638 (+16 ms)Running coprocessor post-open hooks at 1732014336643 (+5 ms)Region opened successfully at 1732014336644 (+1 ms) 2024-11-19T11:05:36,651 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732014336548 2024-11-19T11:05:36,661 DEBUG [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T11:05:36,661 INFO [RS_OPEN_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T11:05:36,663 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:36,665 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c28903dc2d16,43005,1732014332227, state=OPEN 2024-11-19T11:05:36,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-19T11:05:36,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-19T11:05:36,724 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-19T11:05:36,724 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-19T11:05:36,725 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:36,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T11:05:36,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c28903dc2d16,43005,1732014332227 in 343 msec 2024-11-19T11:05:36,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T11:05:36,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 611 msec 2024-11-19T11:05:36,740 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T11:05:36,740 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T11:05:36,758 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:36,759 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:36,778 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:36,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51793, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:05:36,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6470 sec 2024-11-19T11:05:36,808 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732014336807, completionTime=-1 2024-11-19T11:05:36,810 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T11:05:36,810 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T11:05:36,836 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T11:05:36,836 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732014396836 2024-11-19T11:05:36,837 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732014456837 2024-11-19T11:05:36,837 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-19T11:05:36,839 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,39427,1732014330695-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,839 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,39427,1732014330695-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,839 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,39427,1732014330695-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,841 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c28903dc2d16:39427, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,841 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,842 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,848 DEBUG [master/c28903dc2d16:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T11:05:36,876 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.402sec 2024-11-19T11:05:36,877 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T11:05:36,878 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T11:05:36,879 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T11:05:36,880 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T11:05:36,880 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T11:05:36,880 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,39427,1732014330695-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T11:05:36,881 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,39427,1732014330695-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T11:05:36,890 DEBUG [master/c28903dc2d16:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T11:05:36,891 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T11:05:36,891 INFO [master/c28903dc2d16:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28903dc2d16,39427,1732014330695-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T11:05:36,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f28f584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:36,911 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T11:05:36,911 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T11:05:36,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:05:36,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:05:36,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:05:36,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:05:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:05:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c63ae06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:05:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:05:36,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:36,926 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:05:36,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5365a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:36,928 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:36,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:36,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:05:36,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c28903dc2d16,39427,1732014330695 2024-11-19T11:05:36,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-19T11:05:36,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/test.cache.data in system properties and HBase conf 2024-11-19T11:05:36,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T11:05:36,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir in system properties and HBase conf 2024-11-19T11:05:36,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T11:05:36,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T11:05:36,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T11:05:36,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T11:05:36,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T11:05:36,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T11:05:36,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/nfs.dump.dir in system properties and HBase conf 2024-11-19T11:05:36,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir in system properties and HBase conf 2024-11-19T11:05:36,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T11:05:36,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T11:05:36,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T11:05:37,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741835_1011 (size=592039) 2024-11-19T11:05:37,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741836_1012 (size=1663647) 2024-11-19T11:05:38,923 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:39,003 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:39,213 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:05:39,214 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-19T11:05:39,215 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:05:39,215 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:39,215 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:39,215 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T11:05:39,225 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:39,225 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:39,226 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T11:05:39,227 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68f08892{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:39,227 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:39,227 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d8c4596{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-19T11:05:39,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a1040a0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:39,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f41edce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-19T11:05:39,352 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-11-19T11:05:39,352 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-11-19T11:05:39,352 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-19T11:05:39,354 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-19T11:05:39,397 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:39,442 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:39,649 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-11-19T11:05:39,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17ec89fc{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir/jetty-localhost-41879-hadoop-yarn-common-3_4_1_jar-_-any-11577931940602929644/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-19T11:05:39,669 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f51081{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir/jetty-localhost-36575-hadoop-yarn-common-3_4_1_jar-_-any-12719944423856821678/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-19T11:05:39,670 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@403cdf84{HTTP/1.1, (http/1.1)}{localhost:41879} 2024-11-19T11:05:39,670 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@4c920e7f{HTTP/1.1, (http/1.1)}{localhost:36575} 2024-11-19T11:05:39,670 INFO [Time-limited test {}] server.Server(415): Started @15410ms 2024-11-19T11:05:39,670 INFO [Thread-251 {}] server.Server(415): Started @15410ms 2024-11-19T11:05:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741837_1013 (size=5) 2024-11-19T11:05:40,695 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-19T11:05:40,702 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:40,740 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-19T11:05:40,741 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:05:40,746 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:40,746 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:40,746 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T11:05:40,747 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:40,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9e3e547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:40,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b9502e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-19T11:05:40,820 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-19T11:05:40,820 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-19T11:05:40,820 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-19T11:05:40,820 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-19T11:05:40,833 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:40,842 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:40,959 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:40,974 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d2f5bb7{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir/jetty-localhost-41629-hadoop-yarn-common-3_4_1_jar-_-any-11661064785588224455/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-19T11:05:40,975 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b9922e{HTTP/1.1, (http/1.1)}{localhost:41629} 2024-11-19T11:05:40,975 INFO [Time-limited test {}] server.Server(415): Started @16715ms 2024-11-19T11:05:41,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T11:05:41,060 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T11:05:41,069 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:05:41,070 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-19T11:05:41,070 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T11:05:41,070 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T11:05:41,070 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T11:05:41,071 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T11:05:41,100 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-19T11:05:41,103 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:41,115 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-19T11:05:41,115 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T11:05:41,120 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T11:05:41,121 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T11:05:41,121 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T11:05:41,121 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T11:05:41,122 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63b686e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,AVAILABLE} 2024-11-19T11:05:41,122 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@942d7cc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-19T11:05:41,165 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-19T11:05:41,165 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-19T11:05:41,165 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-19T11:05:41,165 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-19T11:05:41,176 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:41,182 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:41,323 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-19T11:05:41,328 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7742a1dc{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/java.io.tmpdir/jetty-localhost-46161-hadoop-yarn-common-3_4_1_jar-_-any-10652310273666532879/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-19T11:05:41,329 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@42cd877f{HTTP/1.1, (http/1.1)}{localhost:46161} 2024-11-19T11:05:41,329 INFO [Time-limited test {}] server.Server(415): Started @17069ms 2024-11-19T11:05:41,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-19T11:05:41,354 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:35831/backupUT 2024-11-19T11:05:41,358 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:05:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c28903dc2d16,39427,1732014330695 2024-11-19T11:05:41,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2201fc65 2024-11-19T11:05:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:05:41,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:05:41,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-19T11:05:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-19T11:05:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:05:41,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 21 msec 2024-11-19T11:05:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T11:05:41,503 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-19T11:05:41,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-19T11:05:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-19T11:05:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-19T11:05:41,520 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T11:05:41,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-11-19T11:05:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-19T11:05:41,613 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-19T11:05:41,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-19T11:05:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-19T11:05:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-19T11:05:41,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 14 msec 2024-11-19T11:05:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-19T11:05:41,734 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-19T11:05:41,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-19T11:05:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-19T11:05:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:05:41,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 14 msec 2024-11-19T11:05:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T11:05:41,843 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-19T11:05:41,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1732014341354', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732014341354 2024-11-19T11:05:41,862 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732014341354 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:05:41,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1732014341354" procId is: 8 2024-11-19T11:05:41,865 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-19T11:05:41,868 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732014341354 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:05:41,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741838_1014 (size=367) 2024-11-19T11:05:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-19T11:05:42,140 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-19T11:05:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-19T11:05:42,284 INFO [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 295075fcca1bb094ad7be004ade119ff, NAME => 'test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732014341354', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741839_1015 (size=53) 2024-11-19T11:05:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-19T11:05:42,695 DEBUG [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:42,695 DEBUG [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1722): Closing 295075fcca1bb094ad7be004ade119ff, disabling compactions & flushes 2024-11-19T11:05:42,695 INFO [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:42,695 DEBUG [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:42,695 DEBUG [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. after waiting 0 ms 2024-11-19T11:05:42,695 DEBUG [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:42,695 INFO [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1973): Closed test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:42,695 DEBUG [RegionOpenAndInit-test-1732014341354-pool-0 {}] regionserver.HRegion(1676): Region close journal for 295075fcca1bb094ad7be004ade119ff: Waiting for close lock at 1732014342695Disabling compacts and flushes for region at 1732014342695Disabling writes for close at 1732014342695Writing region close event to WAL at 1732014342695Closed at 1732014342695 2024-11-19T11:05:42,698 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732014341354 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:05:42,703 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732014342698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014342698"}]},"ts":"1732014342698"} 2024-11-19T11:05:42,708 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:05:42,709 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732014341354 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:05:42,712 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732014341354","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014342710"}]},"ts":"1732014342710"} 2024-11-19T11:05:42,717 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732014341354, state=ENABLING in hbase:meta 2024-11-19T11:05:42,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, ASSIGN}] 2024-11-19T11:05:42,721 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, ASSIGN 2024-11-19T11:05:42,722 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, ASSIGN; state=OFFLINE, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=false 2024-11-19T11:05:42,874 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:42,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, ASSIGN because future has completed 2024-11-19T11:05:42,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-19T11:05:43,055 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:43,055 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T11:05:43,055 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor c28903dc2d16,43005,1732014332227.regiongroup-1 2024-11-19T11:05:43,058 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1, suffix=, logDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227, archiveDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs, maxLogs=32 2024-11-19T11:05:43,073 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060, exclude list is [], retry=0 2024-11-19T11:05:43,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:43,079 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060 2024-11-19T11:05:43,080 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:43,080 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 295075fcca1bb094ad7be004ade119ff, NAME => 'test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:43,080 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:43,081 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732014341354 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,081 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:43,081 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,081 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,083 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,086 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName f 2024-11-19T11:05:43,086 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:43,087 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:43,087 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,088 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,089 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,090 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,090 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,093 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,096 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:43,097 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 295075fcca1bb094ad7be004ade119ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10846471680, jitterRate=0.010156393051147461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T11:05:43,097 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:43,099 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 295075fcca1bb094ad7be004ade119ff: Running coprocessor pre-open hook at 1732014343081Writing region info on filesystem at 1732014343081Initializing all the Stores at 1732014343083 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014343083Cleaning up temporary data from old regions at 1732014343090 (+7 ms)Running coprocessor post-open hooks at 1732014343097 (+7 ms)Region opened successfully at 1732014343098 (+1 ms) 2024-11-19T11:05:43,100 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., pid=10, masterSystemTime=1732014343034 2024-11-19T11:05:43,104 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:43,104 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:43,105 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:43,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:43,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T11:05:43,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 in 232 msec 2024-11-19T11:05:43,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-19T11:05:43,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, ASSIGN in 398 msec 2024-11-19T11:05:43,122 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732014341354 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:05:43,123 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732014341354","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014343122"}]},"ts":"1732014343122"} 2024-11-19T11:05:43,126 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732014341354, state=ENABLED in hbase:meta 2024-11-19T11:05:43,128 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732014341354 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:05:43,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732014341354 in 1.2740 sec 2024-11-19T11:05:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-19T11:05:44,013 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732014341354 completed 2024-11-19T11:05:44,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20726a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:44,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:05:44,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:05:44,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:05:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:05:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:05:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@602bad53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:05:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:05:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:44,018 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:05:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb9bb52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:44,022 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:44,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:05:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732014341354', row='row0', locateType=CURRENT is [region=test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:05:44,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43005 {}] regionserver.HRegion(8528): writing data to region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. with WAL disabled. Data may be lost in the event of a crash. 2024-11-19T11:05:44,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17320143413541', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17320143413541 2024-11-19T11:05:44,199 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17320143413541 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:05:44,200 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:44,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17320143413541" procId is: 11 2024-11-19T11:05:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:05:44,201 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17320143413541 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:05:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741841_1017 (size=364) 2024-11-19T11:05:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:05:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:05:44,614 INFO [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 649b89cbd07b01704a6e5b30a452786a, NAME => 'ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17320143413541', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:44,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741842_1018 (size=50) 2024-11-19T11:05:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:05:45,023 DEBUG [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:45,023 DEBUG [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1722): Closing 649b89cbd07b01704a6e5b30a452786a, disabling compactions & flushes 2024-11-19T11:05:45,023 INFO [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,023 DEBUG [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,023 DEBUG [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. after waiting 0 ms 2024-11-19T11:05:45,023 DEBUG [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,023 INFO [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,024 DEBUG [RegionOpenAndInit-ns2:test-17320143413541-pool-0 {}] regionserver.HRegion(1676): Region close journal for 649b89cbd07b01704a6e5b30a452786a: Waiting for close lock at 1732014345023Disabling compacts and flushes for region at 1732014345023Disabling writes for close at 1732014345023Writing region close event to WAL at 1732014345023Closed at 1732014345023 2024-11-19T11:05:45,026 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17320143413541 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:05:45,026 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732014345026"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014345026"}]},"ts":"1732014345026"} 2024-11-19T11:05:45,029 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:05:45,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17320143413541 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:05:45,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17320143413541","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014345031"}]},"ts":"1732014345031"} 2024-11-19T11:05:45,034 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17320143413541, state=ENABLING in hbase:meta 2024-11-19T11:05:45,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17320143413541, region=649b89cbd07b01704a6e5b30a452786a, ASSIGN}] 2024-11-19T11:05:45,037 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17320143413541, region=649b89cbd07b01704a6e5b30a452786a, ASSIGN 2024-11-19T11:05:45,038 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17320143413541, region=649b89cbd07b01704a6e5b30a452786a, ASSIGN; state=OFFLINE, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=false 2024-11-19T11:05:45,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=649b89cbd07b01704a6e5b30a452786a, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:45,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17320143413541, region=649b89cbd07b01704a6e5b30a452786a, ASSIGN because future has completed 2024-11-19T11:05:45,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 649b89cbd07b01704a6e5b30a452786a, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:05:45,351 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,351 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 649b89cbd07b01704a6e5b30a452786a, NAME => 'ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:45,352 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:45,352 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17320143413541 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,352 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:45,352 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,352 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,355 INFO [StoreOpener-649b89cbd07b01704a6e5b30a452786a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,357 INFO [StoreOpener-649b89cbd07b01704a6e5b30a452786a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 649b89cbd07b01704a6e5b30a452786a columnFamilyName f 2024-11-19T11:05:45,357 DEBUG [StoreOpener-649b89cbd07b01704a6e5b30a452786a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:45,358 INFO [StoreOpener-649b89cbd07b01704a6e5b30a452786a-1 {}] regionserver.HStore(327): Store=649b89cbd07b01704a6e5b30a452786a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:45,358 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,360 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,360 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,361 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,361 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,364 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,367 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:45,368 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 649b89cbd07b01704a6e5b30a452786a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9813507040, jitterRate=-0.08604593575000763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T11:05:45,368 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:05:45,370 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 649b89cbd07b01704a6e5b30a452786a: Running coprocessor pre-open hook at 1732014345352Writing region info on filesystem at 1732014345352Initializing all the Stores at 1732014345354 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014345354Cleaning up temporary data from old regions at 1732014345361 (+7 ms)Running coprocessor post-open hooks at 1732014345368 (+7 ms)Region opened successfully at 1732014345369 (+1 ms) 2024-11-19T11:05:45,371 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a., pid=13, masterSystemTime=1732014345346 2024-11-19T11:05:45,374 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,374 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:05:45,375 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=649b89cbd07b01704a6e5b30a452786a, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:45,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 649b89cbd07b01704a6e5b30a452786a, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:45,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-19T11:05:45,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 649b89cbd07b01704a6e5b30a452786a, server=c28903dc2d16,43005,1732014332227 in 188 msec 2024-11-19T11:05:45,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T11:05:45,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17320143413541, region=649b89cbd07b01704a6e5b30a452786a, ASSIGN in 350 msec 2024-11-19T11:05:45,391 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17320143413541 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:05:45,391 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17320143413541","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014345391"}]},"ts":"1732014345391"} 2024-11-19T11:05:45,394 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17320143413541, state=ENABLED in hbase:meta 2024-11-19T11:05:45,396 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17320143413541 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:05:45,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17320143413541 in 1.2000 sec 2024-11-19T11:05:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T11:05:46,353 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17320143413541 completed 2024-11-19T11:05:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17320143413541', row='row0', locateType=CURRENT is [region=ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:05:46,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43005 {}] regionserver.HRegion(8528): writing data to region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. with WAL disabled. Data may be lost in the event of a crash. 2024-11-19T11:05:46,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17320143413542', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17320143413542 2024-11-19T11:05:46,508 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17320143413542 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:05:46,508 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:46,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17320143413542" procId is: 14 2024-11-19T11:05:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-19T11:05:46,510 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17320143413542 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:05:46,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741843_1019 (size=390) 2024-11-19T11:05:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-19T11:05:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-19T11:05:46,921 INFO [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => af7e2bb446c6a1468013d39e5733df3e, NAME => 'ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17320143413542', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:46,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741844_1020 (size=50) 2024-11-19T11:05:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-19T11:05:47,330 DEBUG [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:47,331 DEBUG [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1722): Closing af7e2bb446c6a1468013d39e5733df3e, disabling compactions & flushes 2024-11-19T11:05:47,331 INFO [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,331 DEBUG [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,331 DEBUG [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. after waiting 0 ms 2024-11-19T11:05:47,331 DEBUG [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,331 INFO [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,331 DEBUG [RegionOpenAndInit-ns3:test-17320143413542-pool-0 {}] regionserver.HRegion(1676): Region close journal for af7e2bb446c6a1468013d39e5733df3e: Waiting for close lock at 1732014347331Disabling compacts and flushes for region at 1732014347331Disabling writes for close at 1732014347331Writing region close event to WAL at 1732014347331Closed at 1732014347331 2024-11-19T11:05:47,333 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17320143413542 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:05:47,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732014347333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014347333"}]},"ts":"1732014347333"} 2024-11-19T11:05:47,336 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:05:47,338 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17320143413542 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:05:47,338 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17320143413542","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014347338"}]},"ts":"1732014347338"} 2024-11-19T11:05:47,341 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17320143413542, state=ENABLING in hbase:meta 2024-11-19T11:05:47,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17320143413542, region=af7e2bb446c6a1468013d39e5733df3e, ASSIGN}] 2024-11-19T11:05:47,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17320143413542, region=af7e2bb446c6a1468013d39e5733df3e, ASSIGN 2024-11-19T11:05:47,344 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17320143413542, region=af7e2bb446c6a1468013d39e5733df3e, ASSIGN; state=OFFLINE, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=false 2024-11-19T11:05:47,495 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=af7e2bb446c6a1468013d39e5733df3e, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:47,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17320143413542, region=af7e2bb446c6a1468013d39e5733df3e, ASSIGN because future has completed 2024-11-19T11:05:47,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure af7e2bb446c6a1468013d39e5733df3e, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:47,523 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732014341354' 2024-11-19T11:05:47,524 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17320143413541' 2024-11-19T11:05:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-19T11:05:47,658 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,658 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => af7e2bb446c6a1468013d39e5733df3e, NAME => 'ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:47,659 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:47,659 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17320143413542 af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,659 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:47,659 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,659 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,661 INFO [StoreOpener-af7e2bb446c6a1468013d39e5733df3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,663 INFO [StoreOpener-af7e2bb446c6a1468013d39e5733df3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af7e2bb446c6a1468013d39e5733df3e columnFamilyName f 2024-11-19T11:05:47,664 DEBUG [StoreOpener-af7e2bb446c6a1468013d39e5733df3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:47,664 INFO [StoreOpener-af7e2bb446c6a1468013d39e5733df3e-1 {}] regionserver.HStore(327): Store=af7e2bb446c6a1468013d39e5733df3e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:47,665 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,666 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns3/test-17320143413542/af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,666 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns3/test-17320143413542/af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,667 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,667 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,670 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,673 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns3/test-17320143413542/af7e2bb446c6a1468013d39e5733df3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:47,673 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened af7e2bb446c6a1468013d39e5733df3e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10750550080, jitterRate=0.0012229979038238525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T11:05:47,674 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:05:47,675 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for af7e2bb446c6a1468013d39e5733df3e: Running coprocessor pre-open hook at 1732014347659Writing region info on filesystem at 1732014347659Initializing all the Stores at 1732014347661 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014347661Cleaning up temporary data from old regions at 1732014347667 (+6 ms)Running coprocessor post-open hooks at 1732014347674 (+7 ms)Region opened successfully at 1732014347675 (+1 ms) 2024-11-19T11:05:47,676 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e., pid=16, masterSystemTime=1732014347653 2024-11-19T11:05:47,679 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,679 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:05:47,680 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=af7e2bb446c6a1468013d39e5733df3e, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:47,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure af7e2bb446c6a1468013d39e5733df3e, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:47,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T11:05:47,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure af7e2bb446c6a1468013d39e5733df3e, server=c28903dc2d16,43005,1732014332227 in 186 msec 2024-11-19T11:05:47,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-19T11:05:47,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17320143413542, region=af7e2bb446c6a1468013d39e5733df3e, ASSIGN in 379 msec 2024-11-19T11:05:47,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17320143413542 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:05:47,724 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17320143413542","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014347724"}]},"ts":"1732014347724"} 2024-11-19T11:05:47,727 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17320143413542, state=ENABLED in hbase:meta 2024-11-19T11:05:47,728 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17320143413542 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:05:47,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17320143413542 in 1.2230 sec 2024-11-19T11:05:48,584 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-19T11:05:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-19T11:05:48,663 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17320143413542 completed 2024-11-19T11:05:48,664 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17320143413542 get assigned. Timeout = 60000ms 2024-11-19T11:05:48,665 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T11:05:48,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17320143413542 assigned to meta. Checking AM states. 2024-11-19T11:05:48,671 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T11:05:48,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17320143413542 assigned. 2024-11-19T11:05:48,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17320143413543', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17320143413543 2024-11-19T11:05:48,676 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17320143413543 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:05:48,676 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:48,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17320143413543" procId is: 17 2024-11-19T11:05:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:05:48,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17320143413543 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:05:48,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741845_1021 (size=390) 2024-11-19T11:05:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:05:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:05:49,089 INFO [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e525c2317c76643ca583f53f5308f38, NAME => 'ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17320143413543', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:49,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741846_1022 (size=50) 2024-11-19T11:05:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:05:49,498 DEBUG [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:49,498 DEBUG [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1722): Closing 5e525c2317c76643ca583f53f5308f38, disabling compactions & flushes 2024-11-19T11:05:49,498 INFO [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,498 DEBUG [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,498 DEBUG [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. after waiting 0 ms 2024-11-19T11:05:49,498 DEBUG [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,498 INFO [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,498 DEBUG [RegionOpenAndInit-ns4:test-17320143413543-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e525c2317c76643ca583f53f5308f38: Waiting for close lock at 1732014349498Disabling compacts and flushes for region at 1732014349498Disabling writes for close at 1732014349498Writing region close event to WAL at 1732014349498Closed at 1732014349498 2024-11-19T11:05:49,500 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17320143413543 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:05:49,500 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732014349500"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014349500"}]},"ts":"1732014349500"} 2024-11-19T11:05:49,503 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:05:49,504 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17320143413543 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:05:49,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17320143413543","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014349504"}]},"ts":"1732014349504"} 2024-11-19T11:05:49,507 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17320143413543, state=ENABLING in hbase:meta 2024-11-19T11:05:49,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17320143413543, region=5e525c2317c76643ca583f53f5308f38, ASSIGN}] 2024-11-19T11:05:49,509 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17320143413543, region=5e525c2317c76643ca583f53f5308f38, ASSIGN 2024-11-19T11:05:49,510 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17320143413543, region=5e525c2317c76643ca583f53f5308f38, ASSIGN; state=OFFLINE, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=false 2024-11-19T11:05:49,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=5e525c2317c76643ca583f53f5308f38, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:49,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17320143413543, region=5e525c2317c76643ca583f53f5308f38, ASSIGN because future has completed 2024-11-19T11:05:49,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e525c2317c76643ca583f53f5308f38, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:05:49,822 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,822 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e525c2317c76643ca583f53f5308f38, NAME => 'ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:49,823 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:49,823 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17320143413543 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,823 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:49,823 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,823 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,825 INFO [StoreOpener-5e525c2317c76643ca583f53f5308f38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,829 INFO [StoreOpener-5e525c2317c76643ca583f53f5308f38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e525c2317c76643ca583f53f5308f38 columnFamilyName f 2024-11-19T11:05:49,829 DEBUG [StoreOpener-5e525c2317c76643ca583f53f5308f38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:49,830 INFO [StoreOpener-5e525c2317c76643ca583f53f5308f38-1 {}] regionserver.HStore(327): Store=5e525c2317c76643ca583f53f5308f38/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:49,830 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,832 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns4/test-17320143413543/5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,832 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns4/test-17320143413543/5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,833 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,833 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,835 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,838 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns4/test-17320143413543/5e525c2317c76643ca583f53f5308f38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:49,839 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 5e525c2317c76643ca583f53f5308f38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12035191360, jitterRate=0.12086454033851624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T11:05:49,839 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:05:49,840 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 5e525c2317c76643ca583f53f5308f38: Running coprocessor pre-open hook at 1732014349823Writing region info on filesystem at 1732014349823Initializing all the Stores at 1732014349825 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014349825Cleaning up temporary data from old regions at 1732014349833 (+8 ms)Running coprocessor post-open hooks at 1732014349839 (+6 ms)Region opened successfully at 1732014349839 2024-11-19T11:05:49,841 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38., pid=19, masterSystemTime=1732014349817 2024-11-19T11:05:49,844 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,844 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:05:49,845 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=5e525c2317c76643ca583f53f5308f38, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:49,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e525c2317c76643ca583f53f5308f38, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:49,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-19T11:05:49,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e525c2317c76643ca583f53f5308f38, server=c28903dc2d16,43005,1732014332227 in 186 msec 2024-11-19T11:05:49,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T11:05:49,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17320143413543, region=5e525c2317c76643ca583f53f5308f38, ASSIGN in 347 msec 2024-11-19T11:05:49,859 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17320143413543 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:05:49,859 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17320143413543","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014349859"}]},"ts":"1732014349859"} 2024-11-19T11:05:49,862 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17320143413543, state=ENABLED in hbase:meta 2024-11-19T11:05:49,864 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17320143413543 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:05:49,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17320143413543 in 1.1910 sec 2024-11-19T11:05:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T11:05:50,823 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17320143413543 completed 2024-11-19T11:05:50,823 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17320143413543 get assigned. Timeout = 60000ms 2024-11-19T11:05:50,824 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T11:05:50,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17320143413543 assigned to meta. Checking AM states. 2024-11-19T11:05:50,828 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T11:05:50,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17320143413543 assigned. 2024-11-19T11:05:50,829 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:05:50,829 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:05:50,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:50,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:50,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:05:50,856 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=610, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=9, AvailableMemoryMB=14752 2024-11-19T11:05:50,857 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=610 is superior to 500 2024-11-19T11:05:50,859 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-19T11:05:50,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732014341354 from 'test-1732014341354', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732014341354', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732014341354 2024-11-19T11:05:50,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741847_1023 (size=908) 2024-11-19T11:05:50,891 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000001.367 2024-11-19T11:05:50,892 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000002.908 2024-11-19T11:05:50,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732014341354}] 2024-11-19T11:05:50,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE}] 2024-11-19T11:05:50,906 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE 2024-11-19T11:05:50,908 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=CLOSING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:50,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE because future has completed 2024-11-19T11:05:50,911 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:05:50,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:51,059 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17320143413542 2024-11-19T11:05:51,059 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17320143413542 Metrics about Tables on a single HBase RegionServer 2024-11-19T11:05:51,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17320143413543 2024-11-19T11:05:51,061 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17320143413543 Metrics about Tables on a single HBase RegionServer 2024-11-19T11:05:51,062 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732014341354 2024-11-19T11:05:51,062 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732014341354 Metrics about Tables on a single HBase RegionServer 2024-11-19T11:05:51,063 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17320143413541 2024-11-19T11:05:51,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17320143413541 Metrics about Tables on a single HBase RegionServer 2024-11-19T11:05:51,070 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,070 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:05:51,071 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 295075fcca1bb094ad7be004ade119ff, disabling compactions & flushes 2024-11-19T11:05:51,071 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:51,071 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:51,071 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. after waiting 0 ms 2024-11-19T11:05:51,071 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:51,075 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 295075fcca1bb094ad7be004ade119ff 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-19T11:05:51,144 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f/eb6e11f7dbd740978d1356b438bf56b3 is 37, key is row10/f:q1/1732014344053/Put/seqid=0 2024-11-19T11:05:51,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741848_1024 (size=8413) 2024-11-19T11:05:51,555 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f/eb6e11f7dbd740978d1356b438bf56b3 2024-11-19T11:05:51,612 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f/eb6e11f7dbd740978d1356b438bf56b3 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 2024-11-19T11:05:51,622 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3, entries=99, sequenceid=103, filesize=8.2 K 2024-11-19T11:05:51,629 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 295075fcca1bb094ad7be004ade119ff in 554ms, sequenceid=103, compaction requested=false 2024-11-19T11:05:51,635 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-19T11:05:51,638 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:05:51,638 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:51,639 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 295075fcca1bb094ad7be004ade119ff: Waiting for close lock at 1732014351071Running coprocessor pre-close hooks at 1732014351071Disabling compacts and flushes for region at 1732014351071Disabling writes for close at 1732014351071Obtaining lock to block concurrent updates at 1732014351075 (+4 ms)Preparing flush snapshotting stores in 295075fcca1bb094ad7be004ade119ff at 1732014351075Finished memstore snapshotting test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732014351082 (+7 ms)Flushing stores of test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. at 1732014351084 (+2 ms)Flushing 295075fcca1bb094ad7be004ade119ff/f: creating writer at 1732014351085 (+1 ms)Flushing 295075fcca1bb094ad7be004ade119ff/f: appending metadata at 1732014351129 (+44 ms)Flushing 295075fcca1bb094ad7be004ade119ff/f: closing flushed file at 1732014351133 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5761ae85: reopening flushed file at 1732014351610 (+477 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 295075fcca1bb094ad7be004ade119ff in 554ms, sequenceid=103, compaction requested=false at 1732014351629 (+19 ms)Writing region close event to WAL at 1732014351630 (+1 ms)Running coprocessor post-close hooks at 1732014351636 (+6 ms)Closed at 1732014351638 (+2 ms) 2024-11-19T11:05:51,639 WARN [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 295075fcca1bb094ad7be004ade119ff to self. 2024-11-19T11:05:51,642 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=CLOSED 2024-11-19T11:05:51,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:51,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-19T11:05:51,650 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 in 736 msec 2024-11-19T11:05:51,651 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE; state=CLOSED, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=true 2024-11-19T11:05:51,802 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:51,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE because future has completed 2024-11-19T11:05:51,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:51,963 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:51,963 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 295075fcca1bb094ad7be004ade119ff, NAME => 'test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:51,964 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:51,964 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732014341354 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,964 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:51,964 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,964 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,966 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,967 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName f 2024-11-19T11:05:51,968 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:51,979 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 2024-11-19T11:05:51,979 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:51,979 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,982 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName f3 2024-11-19T11:05:51,982 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:51,983 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:51,984 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,991 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName mob 2024-11-19T11:05:51,996 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:51,997 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:51,998 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:51,999 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:52,002 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:52,004 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:52,004 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:52,005 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732014341354 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T11:05:52,008 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:52,009 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 295075fcca1bb094ad7be004ade119ff; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10247011200, jitterRate=-0.045672714710235596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T11:05:52,009 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:52,010 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 295075fcca1bb094ad7be004ade119ff: Running coprocessor pre-open hook at 1732014351964Writing region info on filesystem at 1732014351964Initializing all the Stores at 1732014351966 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014351966Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014351966Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014351966Cleaning up temporary data from old regions at 1732014352004 (+38 ms)Running coprocessor post-open hooks at 1732014352009 (+5 ms)Region opened successfully at 1732014352010 (+1 ms) 2024-11-19T11:05:52,011 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., pid=24, masterSystemTime=1732014351958 2024-11-19T11:05:52,014 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:52,014 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:52,015 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=OPEN, openSeqNum=107, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:52,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:52,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-19T11:05:52,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 in 213 msec 2024-11-19T11:05:52,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T11:05:52,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE in 1.1170 sec 2024-11-19T11:05:52,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-19T11:05:52,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732014341354 in 1.1310 sec 2024-11-19T11:05:52,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732014341354 in 1.1670 sec 2024-11-19T11:05:52,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-19T11:05:52,038 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732014341354 completed 2024-11-19T11:05:52,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361bac50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:52,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:05:52,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:05:52,040 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:05:52,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:05:52,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:05:52,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d39f167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:52,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:05:52,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:05:52,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:52,042 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:05:52,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c43aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:52,044 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:52,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:52,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:52,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:05:52,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732014341354', row='row-test-1732014341354-3-0', locateType=CURRENT is [region=test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., hostname=c28903dc2d16,43005,1732014332227, seqNum=107] 2024-11-19T11:05:52,106 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:05:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is c28903dc2d16,39427,1732014330695 2024-11-19T11:05:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5382eb8 2024-11-19T11:05:52,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:05:52,111 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:05:52,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-19T11:05:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-19T11:05:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:05:52,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 11 msec 2024-11-19T11:05:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-19T11:05:52,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-19T11:05:52,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-19T11:05:52,256 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:05:52,256 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:52,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-19T11:05:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T11:05:52,258 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:05:52,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741849_1025 (size=635) 2024-11-19T11:05:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T11:05:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T11:05:52,669 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9eb809ed54235dbcfb549c070333cd60, NAME => 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:52,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741850_1026 (size=40) 2024-11-19T11:05:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T11:05:53,078 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:53,078 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 9eb809ed54235dbcfb549c070333cd60, disabling compactions & flushes 2024-11-19T11:05:53,078 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,078 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,078 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. after waiting 0 ms 2024-11-19T11:05:53,078 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,078 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,079 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9eb809ed54235dbcfb549c070333cd60: Waiting for close lock at 1732014353078Disabling compacts and flushes for region at 1732014353078Disabling writes for close at 1732014353078Writing region close event to WAL at 1732014353078Closed at 1732014353078 2024-11-19T11:05:53,081 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:05:53,081 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732014353081"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014353081"}]},"ts":"1732014353081"} 2024-11-19T11:05:53,084 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:05:53,085 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:05:53,085 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014353085"}]},"ts":"1732014353085"} 2024-11-19T11:05:53,088 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-19T11:05:53,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN}] 2024-11-19T11:05:53,090 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN 2024-11-19T11:05:53,091 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN; state=OFFLINE, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=false 2024-11-19T11:05:53,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9eb809ed54235dbcfb549c070333cd60, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:53,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN because future has completed 2024-11-19T11:05:53,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T11:05:53,403 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,404 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 9eb809ed54235dbcfb549c070333cd60, NAME => 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:53,404 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:53,404 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,404 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:53,405 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,405 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,407 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,410 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eb809ed54235dbcfb549c070333cd60 columnFamilyName meta 2024-11-19T11:05:53,410 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:53,411 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(327): Store=9eb809ed54235dbcfb549c070333cd60/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:53,411 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,413 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eb809ed54235dbcfb549c070333cd60 columnFamilyName session 2024-11-19T11:05:53,413 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:53,414 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(327): Store=9eb809ed54235dbcfb549c070333cd60/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:53,414 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,415 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,416 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,417 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,417 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,418 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T11:05:53,419 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,423 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:53,423 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 9eb809ed54235dbcfb549c070333cd60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10097041440, jitterRate=-0.059639737010002136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T11:05:53,423 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:53,424 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 9eb809ed54235dbcfb549c070333cd60: Running coprocessor pre-open hook at 1732014353405Writing region info on filesystem at 1732014353405Initializing all the Stores at 1732014353407 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014353407Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014353407Cleaning up temporary data from old regions at 1732014353417 (+10 ms)Running coprocessor post-open hooks at 1732014353423 (+6 ms)Region opened successfully at 1732014353424 (+1 ms) 2024-11-19T11:05:53,425 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., pid=28, masterSystemTime=1732014353398 2024-11-19T11:05:53,428 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,428 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:53,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9eb809ed54235dbcfb549c070333cd60, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:53,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:53,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-19T11:05:53,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227 in 188 msec 2024-11-19T11:05:53,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-19T11:05:53,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN in 348 msec 2024-11-19T11:05:53,441 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:05:53,441 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014353441"}]},"ts":"1732014353441"} 2024-11-19T11:05:53,444 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-19T11:05:53,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:05:53,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1930 sec 2024-11-19T11:05:53,526 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17320143413543' 2024-11-19T11:05:53,526 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17320143413542' 2024-11-19T11:05:53,527 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-19T11:05:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-19T11:05:54,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-19T11:05:54,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:05:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-19T11:05:54,422 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T11:05:54,423 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:54,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-19T11:05:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:05:54,424 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T11:05:54,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741851_1027 (size=640) 2024-11-19T11:05:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:05:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:05:54,835 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1b8b050c5202951d3c6975654ccd6c4, NAME => 'backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:05:54,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741852_1028 (size=45) 2024-11-19T11:05:54,845 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:54,845 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing e1b8b050c5202951d3c6975654ccd6c4, disabling compactions & flushes 2024-11-19T11:05:54,845 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:54,845 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:54,845 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. after waiting 0 ms 2024-11-19T11:05:54,845 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:54,845 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:54,845 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1b8b050c5202951d3c6975654ccd6c4: Waiting for close lock at 1732014354845Disabling compacts and flushes for region at 1732014354845Disabling writes for close at 1732014354845Writing region close event to WAL at 1732014354845Closed at 1732014354845 2024-11-19T11:05:54,847 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T11:05:54,847 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732014354847"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014354847"}]},"ts":"1732014354847"} 2024-11-19T11:05:54,850 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:05:54,852 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T11:05:54,852 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014354852"}]},"ts":"1732014354852"} 2024-11-19T11:05:54,855 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-19T11:05:54,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=e1b8b050c5202951d3c6975654ccd6c4, ASSIGN}] 2024-11-19T11:05:54,857 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=e1b8b050c5202951d3c6975654ccd6c4, ASSIGN 2024-11-19T11:05:54,858 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=e1b8b050c5202951d3c6975654ccd6c4, ASSIGN; state=OFFLINE, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=false 2024-11-19T11:05:55,009 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e1b8b050c5202951d3c6975654ccd6c4, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:55,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=e1b8b050c5202951d3c6975654ccd6c4, ASSIGN because future has completed 2024-11-19T11:05:55,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1b8b050c5202951d3c6975654ccd6c4, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:05:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:05:55,170 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:55,170 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e1b8b050c5202951d3c6975654ccd6c4, NAME => 'backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:05:55,170 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:05:55,170 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,170 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:05:55,170 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,171 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,173 INFO [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,175 INFO [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1b8b050c5202951d3c6975654ccd6c4 columnFamilyName meta 2024-11-19T11:05:55,175 DEBUG [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:55,175 INFO [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] regionserver.HStore(327): Store=e1b8b050c5202951d3c6975654ccd6c4/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:55,176 INFO [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,177 INFO [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1b8b050c5202951d3c6975654ccd6c4 columnFamilyName session 2024-11-19T11:05:55,177 DEBUG [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:05:55,178 INFO [StoreOpener-e1b8b050c5202951d3c6975654ccd6c4-1 {}] regionserver.HStore(327): Store=e1b8b050c5202951d3c6975654ccd6c4/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:05:55,178 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,179 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system_bulk/e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,180 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system_bulk/e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,181 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,181 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,181 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T11:05:55,183 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,186 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system_bulk/e1b8b050c5202951d3c6975654ccd6c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T11:05:55,187 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e1b8b050c5202951d3c6975654ccd6c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10031361120, jitterRate=-0.06575669348239899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T11:05:55,187 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:05:55,187 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e1b8b050c5202951d3c6975654ccd6c4: Running coprocessor pre-open hook at 1732014355171Writing region info on filesystem at 1732014355171Initializing all the Stores at 1732014355172 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014355172Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014355173 (+1 ms)Cleaning up temporary data from old regions at 1732014355181 (+8 ms)Running coprocessor post-open hooks at 1732014355187 (+6 ms)Region opened successfully at 1732014355187 2024-11-19T11:05:55,189 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4., pid=31, masterSystemTime=1732014355165 2024-11-19T11:05:55,191 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:55,192 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:05:55,192 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e1b8b050c5202951d3c6975654ccd6c4, regionState=OPEN, openSeqNum=2, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:05:55,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1b8b050c5202951d3c6975654ccd6c4, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:05:55,196 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39427 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=c28903dc2d16,43005,1732014332227, table=backup:system_bulk, region=e1b8b050c5202951d3c6975654ccd6c4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T11:05:55,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T11:05:55,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e1b8b050c5202951d3c6975654ccd6c4, server=c28903dc2d16,43005,1732014332227 in 185 msec 2024-11-19T11:05:55,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T11:05:55,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=e1b8b050c5202951d3c6975654ccd6c4, ASSIGN in 346 msec 2024-11-19T11:05:55,206 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T11:05:55,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014355206"}]},"ts":"1732014355206"} 2024-11-19T11:05:55,209 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-19T11:05:55,210 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T11:05:55,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 791 msec 2024-11-19T11:05:55,326 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-19T11:05:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-19T11:05:55,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-19T11:05:55,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:05:55,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:05:55,590 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-19T11:05:55,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:05:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43005 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-19T11:05:55,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:05:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014355627 (current time:1732014355627). 2024-11-19T11:05:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:05:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-19T11:05:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b6180d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:05:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:05:55,630 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:05:55,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:05:55,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:05:55,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ff4920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:55,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:05:55,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:05:55,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:55,633 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:05:55,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596e130b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:55,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:55,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:55,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:05:55,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:05:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:05:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:55,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:05:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:05:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:05:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-19T11:05:55,662 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:05:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T11:05:55,669 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:05:55,686 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:05:55,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741853_1029 (size=52) 2024-11-19T11:05:55,717 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:05:55,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60}] 2024-11-19T11:05:55,727 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T11:05:55,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-19T11:05:55,889 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:55,890 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 9eb809ed54235dbcfb549c070333cd60 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-19T11:05:55,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/7b21082dcb0543e5bc0021ac30d34c67 is 49, key is activesession:/session:c/1732014355613/Put/seqid=0 2024-11-19T11:05:55,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741854_1030 (size=5049) 2024-11-19T11:05:55,918 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/7b21082dcb0543e5bc0021ac30d34c67 2024-11-19T11:05:55,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/7b21082dcb0543e5bc0021ac30d34c67 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 2024-11-19T11:05:55,943 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, entries=1, sequenceid=5, filesize=4.9 K 2024-11-19T11:05:55,944 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 54ms, sequenceid=5, compaction requested=false 2024-11-19T11:05:55,945 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 9eb809ed54235dbcfb549c070333cd60: 2024-11-19T11:05:55,945 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. for snapshot_backup_system completed. 2024-11-19T11:05:55,946 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.' region-info for snapshot=snapshot_backup_system 2024-11-19T11:05:55,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:05:55,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-19T11:05:55,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67] hfiles 2024-11-19T11:05:55,955 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 for snapshot=snapshot_backup_system 2024-11-19T11:05:55,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741855_1031 (size=95) 2024-11-19T11:05:55,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T11:05:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T11:05:56,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:05:56,370 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-19T11:05:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-19T11:05:56,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:56,372 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:05:56,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T11:05:56,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 in 652 msec 2024-11-19T11:05:56,376 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:05:56,378 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:05:56,379 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:05:56,379 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-19T11:05:56,382 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-19T11:05:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741856_1032 (size=731) 2024-11-19T11:05:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T11:05:56,803 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:05:56,816 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:05:56,817 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_backup_system 2024-11-19T11:05:56,820 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:05:56,820 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-19T11:05:56,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1670 sec 2024-11-19T11:05:57,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-19T11:05:57,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-19T11:05:57,813 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732014352089 started at 1732014357812. 2024-11-19T11:05:57,827 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732014352089 has been started. 2024-11-19T11:05:57,833 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-19T11:05:57,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-19T11:05:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-19T11:05:57,842 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-19T11:05:57,842 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-19T11:05:57,843 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-19T11:05:57,843 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-19T11:05:57,844 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:05:57,844 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:05:57,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:05:57,955 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,955 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:05:57,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:05:57,956 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,956 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-19T11:05:57,956 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-19T11:05:57,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:05:57,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-19T11:05:57,957 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:05:57,957 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-19T11:05:57,960 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-19T11:05:57,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-19T11:05:57,960 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-19T11:05:57,961 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-19T11:05:57,961 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-19T11:05:57,961 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-19T11:05:57,961 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure (rolllog) in zk 2024-11-19T11:05:57,974 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:57,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:05:57,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:05:57,975 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-19T11:05:57,975 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-19T11:05:57,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:05:57,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:05:57,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:05:57,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:05:57,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-19T11:05:57,979 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-19T11:05:57,979 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@756c6560[Count = 0] remaining members to acquire global barrier 2024-11-19T11:05:57,979 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:57,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:57,986 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:57,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:57,987 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,987 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-19T11:05:57,987 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-19T11:05:57,988 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-19T11:05:57,988 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,994 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732014335713 highest: 1732014343060 on c28903dc2d16,43005,1732014332227 2024-11-19T11:05:57,994 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.meta:.meta(num 1732014336578) roll requested 2024-11-19T11:05:58,007 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta, exclude list is [], retry=0 2024-11-19T11:05:58,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:58,013 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014336578.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta 2024-11-19T11:05:58,014 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:58,014 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014336578.meta is not closed yet, will try archiving it next time 2024-11-19T11:05:58,014 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0:(num 1732014335713) roll requested 2024-11-19T11:05:58,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741834_1010 (size=12055) 2024-11-19T11:05:58,028 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014, exclude list is [], retry=0 2024-11-19T11:05:58,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:58,033 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 2024-11-19T11:05:58,034 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:58,034 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713 is not closed yet, will try archiving it next time 2024-11-19T11:05:58,034 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1:(num 1732014343060) roll requested 2024-11-19T11:05:58,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741833_1009 (size=1034) 2024-11-19T11:05:58,036 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713 not finished, retry = 0 2024-11-19T11:05:58,047 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034, exclude list is [], retry=0 2024-11-19T11:05:58,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:05:58,052 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 2024-11-19T11:05:58,053 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:05:58,053 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060 is not closed yet, will try archiving it next time 2024-11-19T11:05:58,055 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-19T11:05:58,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741840_1016 (size=4756) 2024-11-19T11:05:58,055 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060 not finished, retry = 0 2024-11-19T11:05:58,055 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732014358014 on c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,055 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:05:58,055 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is c28903dc2d16,39427,1732014330695 2024-11-19T11:05:58,055 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@307ae3c6 2024-11-19T11:05:58,056 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:05:58,057 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50141, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-19T11:05:58,058 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:58,058 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:58,058 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:58,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41489, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-19T11:05:58,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:05:58,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:05:58,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:35831/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:05:58,090 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-19T11:05:58,091 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'c28903dc2d16,43005,1732014332227' in zk 2024-11-19T11:05:58,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,102 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-19T11:05:58,102 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:05:58,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:05:58,102 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-19T11:05:58,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:05:58,103 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:05:58,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:05:58,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:05:58,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:05:58,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:05:58,107 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'c28903dc2d16,43005,1732014332227': 2024-11-19T11:05:58,108 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-19T11:05:58,108 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-19T11:05:58,108 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-19T11:05:58,108 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-19T11:05:58,108 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'c28903dc2d16,43005,1732014332227' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-19T11:05:58,108 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-19T11:05:58,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:05:58,122 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:05:58,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:05:58,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:05:58,122 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:05:58,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:05:58,123 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:05:58,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:05:58,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:05:58,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,124 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:05:58,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:05:58,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:05:58,134 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:58,134 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:05:58,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:05:58,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:05:58,135 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:05:58,135 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:05:58,135 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:05:58,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-19T11:05:58,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-19T11:05:58,138 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014335713 2024-11-19T11:05:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-19T11:05:58,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } 2024-11-19T11:05:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014358149 (current time:1732014358149). 2024-11-19T11:05:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:05:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014358148_default_test-1732014341354 VERSION not specified, setting to 2 2024-11-19T11:05:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61418f28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:05:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:05:58,151 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:05:58,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:05:58,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:05:58,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244305c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:58,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:05:58,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:05:58,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:58,152 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:05:58,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127e75c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:05:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:05:58,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:05:58,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:05:58,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:05:58,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:05:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:05:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:58,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:05:58,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:05:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:05:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } 2024-11-19T11:05:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-19T11:05:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:05:58,162 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:05:58,163 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:05:58,167 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:05:58,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741860_1036 (size=84) 2024-11-19T11:05:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:05:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:05:58,577 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:05:58,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff}] 2024-11-19T11:05:58,578 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:05:58,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-19T11:05:58,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:05:58,732 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 295075fcca1bb094ad7be004ade119ff 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-19T11:05:58,749 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f3/c1df863c95e8403b9cac1416030240f9 is 58, key is row-test-1732014341354-3-0/f3:q1/1732014352051/Put/seqid=0 2024-11-19T11:05:58,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741861_1037 (size=5387) 2024-11-19T11:05:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:05:59,155 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f3/c1df863c95e8403b9cac1416030240f9 2024-11-19T11:05:59,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741862_1038 (size=4469) 2024-11-19T11:05:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:05:59,528 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-19T11:05:59,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/e69629895bf34199a73f0520391b1145 is 59, key is row-test-1732014341354-3-0/mob:q1/1732014352069/Put/seqid=0 2024-11-19T11:05:59,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741863_1039 (size=5484) 2024-11-19T11:05:59,695 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:06:00,004 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/e69629895bf34199a73f0520391b1145 2024-11-19T11:06:00,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f3/c1df863c95e8403b9cac1416030240f9 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 2024-11-19T11:06:00,026 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9, entries=6, sequenceid=121, filesize=5.3 K 2024-11-19T11:06:00,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/e69629895bf34199a73f0520391b1145 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 2024-11-19T11:06:00,038 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145, entries=6, sequenceid=121, filesize=5.4 K 2024-11-19T11:06:00,040 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 295075fcca1bb094ad7be004ade119ff in 1309ms, sequenceid=121, compaction requested=false 2024-11-19T11:06:00,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 295075fcca1bb094ad7be004ade119ff: 2024-11-19T11:06:00,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. for snapshot_1732014358148_default_test-1732014341354 completed. 2024-11-19T11:06:00,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.' region-info for snapshot=snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3] hfiles 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 for snapshot=snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9] hfiles 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 for snapshot=snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145] hfiles 2024-11-19T11:06:00,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 for snapshot=snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741864_1040 (size=185) 2024-11-19T11:06:00,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:00,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-19T11:06:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-19T11:06:00,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014358148_default_test-1732014341354 on region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:00,050 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:00,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-19T11:06:00,054 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:00,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff in 1.4740 sec 2024-11-19T11:06:00,056 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:00,058 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-19T11:06:00,058 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-19T11:06:00,058 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:00,059 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-19T11:06:00,059 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:00,060 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-19T11:06:00,060 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:00,060 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-19T11:06:00,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741865_1041 (size=49) 2024-11-19T11:06:00,068 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:00,068 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,069 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741866_1042 (size=1146) 2024-11-19T11:06:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:06:00,513 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:00,527 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:00,528 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014358148_default_test-1732014341354 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:00,530 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:00,530 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-19T11:06:00,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732014358148_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } in 2.3710 sec 2024-11-19T11:06:00,673 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-19T11:06:01,059 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-19T11:06:01,059 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-19T11:06:01,061 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-19T11:06:01,061 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-19T11:06:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-19T11:06:02,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732014341354 completed 2024-11-19T11:06:02,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } 2024-11-19T11:06:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014362343 (current time:1732014362343). 2024-11-19T11:06:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014362343_ns2_test-17320143413541 VERSION not specified, setting to 2 2024-11-19T11:06:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d22975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:02,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:02,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:02,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:02,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b44332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:02,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:02,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:02,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:02,348 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:02,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffe024d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:02,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:02,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:02,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:02,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:02,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } 2024-11-19T11:06:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-19T11:06:02,357 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T11:06:02,358 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:02,362 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:02,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741867_1043 (size=86) 2024-11-19T11:06:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T11:06:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T11:06:02,774 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:02,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a}] 2024-11-19T11:06:02,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:06:02,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-19T11:06:02,929 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:06:02,929 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 649b89cbd07b01704a6e5b30a452786a 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-19T11:06:02,949 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/.tmp/f/b156ab1ca4e541409b66d857a83a7b86 is 37, key is row10/f:q1/1732014346374/Put/seqid=0 2024-11-19T11:06:02,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741868_1044 (size=8413) 2024-11-19T11:06:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T11:06:03,356 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/.tmp/f/b156ab1ca4e541409b66d857a83a7b86 2024-11-19T11:06:03,366 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/.tmp/f/b156ab1ca4e541409b66d857a83a7b86 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 2024-11-19T11:06:03,375 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86, entries=99, sequenceid=103, filesize=8.2 K 2024-11-19T11:06:03,377 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 649b89cbd07b01704a6e5b30a452786a in 447ms, sequenceid=103, compaction requested=false 2024-11-19T11:06:03,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 649b89cbd07b01704a6e5b30a452786a: 2024-11-19T11:06:03,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. for snapshot_1732014362343_ns2_test-17320143413541 completed. 2024-11-19T11:06:03,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.' region-info for snapshot=snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:03,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:03,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86] hfiles 2024-11-19T11:06:03,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 for snapshot=snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:03,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741869_1045 (size=91) 2024-11-19T11:06:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T11:06:03,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:06:03,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-19T11:06:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-19T11:06:03,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014362343_ns2_test-17320143413541 on region 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:06:03,787 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:06:03,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T11:06:03,791 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:03,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a in 1.0140 sec 2024-11-19T11:06:03,792 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:03,793 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:03,793 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:03,794 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:03,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741870_1046 (size=456) 2024-11-19T11:06:04,206 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:04,215 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:04,216 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014362343_ns2_test-17320143413541 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:04,218 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:04,218 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-19T11:06:04,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732014362343_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } in 1.8630 sec 2024-11-19T11:06:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T11:06:04,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17320143413541 completed 2024-11-19T11:06:04,503 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732014352089 2024-11-19T11:06:04,503 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-19T11:06:04,504 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-19T11:06:04,506 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014352089_test-1732014341354 2024-11-19T11:06:04,506 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014358148_default_test-1732014341354 to hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354/ 2024-11-19T11:06:04,510 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:04,543 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:04,543 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354/.hbase-snapshot/.tmp/snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:04,545 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:04,553 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014358148_default_test-1732014341354 to hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354/.hbase-snapshot/.tmp/snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:04,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741872_1048 (size=1146) 2024-11-19T11:06:04,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741871_1047 (size=84) 2024-11-19T11:06:04,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:04,976 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:04,976 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,906 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-6745052073385342365.jar 2024-11-19T11:06:05,907 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,907 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,907 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,908 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,908 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,908 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:05,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:05,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:05,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:05,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:05,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:05,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:05,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:05,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:05,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:05,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:05,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:05,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:05,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:05,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:05,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:05,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:05,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:05,915 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:05,976 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:06,007 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014358148_default_test-1732014341354' hfile list 2024-11-19T11:06:06,016 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=18.8 K 2024-11-19T11:06:06,231 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1267815741_0001_e3c19deb-aa37-43d0-ba91-8f95adf9c021/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:06,232 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1267815741_0001_e3c19deb-aa37-43d0-ba91-8f95adf9c021/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:06,421 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-19T11:06:06,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741873_1049 (size=8413) 2024-11-19T11:06:06,898 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 output=hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 2024-11-19T11:06:06,900 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-11-19T11:06:06,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741874_1050 (size=5484) 2024-11-19T11:06:07,337 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 output=hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 2024-11-19T11:06:07,337 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-19T11:06:07,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741875_1051 (size=5387) 2024-11-19T11:06:07,791 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 output=hdfs://localhost:35831/backupUT/backup_1732014352089/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 2024-11-19T11:06:07,791 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-19T11:06:08,329 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:08,331 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:08,339 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:08,339 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014358148_default_test-1732014341354 finished. 2024-11-19T11:06:08,340 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014352089_ns2:test-17320143413541 2024-11-19T11:06:08,340 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014362343_ns2_test-17320143413541 to hdfs://localhost:35831/backupUT/backup_1732014352089/ns2/test-17320143413541/ 2024-11-19T11:06:08,340 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:08,381 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:08,381 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014352089/ns2/test-17320143413541, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014352089/ns2/test-17320143413541/.hbase-snapshot/.tmp/snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:08,388 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:08,396 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014362343_ns2_test-17320143413541 to hdfs://localhost:35831/backupUT/backup_1732014352089/ns2/test-17320143413541/.hbase-snapshot/.tmp/snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:08,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741876_1052 (size=86) 2024-11-19T11:06:08,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741877_1053 (size=456) 2024-11-19T11:06:08,821 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:08,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:08,822 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,712 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-9460697160185370142.jar 2024-11-19T11:06:09,712 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,713 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,713 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,713 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,714 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,714 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,714 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,715 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,715 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:09,715 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:09,715 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:09,716 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:09,716 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:09,716 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:09,716 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:09,717 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:09,717 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:09,717 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:09,717 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:09,718 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:09,718 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:09,718 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:09,718 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:09,719 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:09,719 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:09,719 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:09,719 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:09,760 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:09,786 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014362343_ns2_test-17320143413541' hfile list 2024-11-19T11:06:09,789 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=8.2 K 2024-11-19T11:06:09,904 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local516068665_0002_1cb20416-136f-494b-800c-b140d7b0ae3e/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:09,904 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local516068665_0002_1cb20416-136f-494b-800c-b140d7b0ae3e/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:09,979 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-19T11:06:10,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741878_1054 (size=8413) 2024-11-19T11:06:10,432 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 output=hdfs://localhost:35831/backupUT/backup_1732014352089/ns2/test-17320143413541/archive/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 2024-11-19T11:06:10,432 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-19T11:06:10,945 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:10,946 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:10,953 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:10,953 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014362343_ns2_test-17320143413541 finished. 2024-11-19T11:06:10,953 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732014341354 2024-11-19T11:06:10,953 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17320143413541 2024-11-19T11:06:11,003 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732014352089 2024-11-19T11:06:11,003 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-19T11:06:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741879_1055 (size=130) 2024-11-19T11:06:11,412 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:35831/backupUT/backup_1732014352089/.backup.manifest 2024-11-19T11:06:11,414 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732014352089 finished: type=FULL,tablelist=test-1732014341354;ns2:test-17320143413541,targetRootDir=hdfs://localhost:35831/backupUT,startts=1732014357812,completets=1732014371001,bytescopied=0 2024-11-19T11:06:11,414 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-19T11:06:11,414 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:11,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014358148_default_test-1732014341354" type: DISABLED 2024-11-19T11:06:11,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014358148_default_test-1732014341354 2024-11-19T11:06:11,424 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014358148_default_test-1732014341354 for backup backup_1732014352089 succeeded. 2024-11-19T11:06:11,425 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:11,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014362343_ns2_test-17320143413541" type: DISABLED 2024-11-19T11:06:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014362343_ns2_test-17320143413541 2024-11-19T11:06:11,429 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014362343_ns2_test-17320143413541 for backup backup_1732014352089 succeeded. 2024-11-19T11:06:11,430 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-19T11:06:11,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-19T11:06:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-19T11:06:11,445 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-19T11:06:11,450 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-19T11:06:11,466 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732014352089 completed. 2024-11-19T11:06:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:11,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732014352089', locateType=CURRENT is [region=backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:06:11,703 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732014341354 2024-11-19T11:06:11,726 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732014341354 2024-11-19T11:06:11,930 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732014341354 to Mob enabled CF 2024-11-19T11:06:11,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17320143413541', row='row-t20', locateType=CURRENT is [region=ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:06:11,965 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17320143413541 2024-11-19T11:06:11,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:11,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] assignment.SplitTableRegionProcedure(210): test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. splitKey isn't explicitly specified, will try to find a best split key from RS c28903dc2d16,43005,1732014332227 2024-11-19T11:06:11,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 because midkey is the same as first or last row 2024-11-19T11:06:11,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 because midkey is the same as first or last row 2024-11-19T11:06:11,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 because midkey is the same as first or last row 2024-11-19T11:06:11,976 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HRegion(2902): Flushing 295075fcca1bb094ad7be004ade119ff 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-19T11:06:11,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f/3a4b0c30a3534ca3824eb1970bbab950 is 59, key is row-test-1732014341354-1-10/f:q1/1732014371522/Put/seqid=0 2024-11-19T11:06:12,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741880_1056 (size=10796) 2024-11-19T11:06:12,401 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f/3a4b0c30a3534ca3824eb1970bbab950 2024-11-19T11:06:12,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff is 63, key is row-test-1732014341354-3-100/mob:q1/1732014371910/Put/seqid=0 2024-11-19T11:06:12,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741881_1057 (size=5591) 2024-11-19T11:06:12,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:12,835 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:12,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/3bc5d5500d8f420f9635501399f50008, store: [table=test-1732014341354 family=mob region=295075fcca1bb094ad7be004ade119ff] 2024-11-19T11:06:12,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/3bc5d5500d8f420f9635501399f50008 is 192, key is row-test-1732014341354-3-100/mob:q1/1732014371910/Put/seqid=0 2024-11-19T11:06:12,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741882_1058 (size=13760) 2024-11-19T11:06:13,242 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/3bc5d5500d8f420f9635501399f50008 2024-11-19T11:06:13,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f/3a4b0c30a3534ca3824eb1970bbab950 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950 2024-11-19T11:06:13,259 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950, entries=99, sequenceid=334, filesize=10.5 K 2024-11-19T11:06:13,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/mob/3bc5d5500d8f420f9635501399f50008 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008 2024-11-19T11:06:13,267 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008, entries=111, sequenceid=334, filesize=13.4 K 2024-11-19T11:06:13,268 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 295075fcca1bb094ad7be004ade119ff in 1292ms, sequenceid=334, compaction requested=false 2024-11-19T11:06:13,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.HRegion(2603): Flush status journal for 295075fcca1bb094ad7be004ade119ff: 2024-11-19T11:06:13,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950 because midkey is the same as first or last row 2024-11-19T11:06:13,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 because midkey is the same as first or last row 2024-11-19T11:06:13,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008 because midkey is the same as first or last row 2024-11-19T11:06:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=c28903dc2d16,43005,1732014332227 2024-11-19T11:06:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T11:06:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.CallRunner(138): callId: 302 service: MasterService methodName: SplitRegion size: 99 connection: 172.17.0.2:53310 deadline: 1732014431971, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-19T11:06:13,275 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:13,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732014341354 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:13,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T11:06:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732014341354,, stopping at row=test-1732014341354 ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,288 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732014341354,, stopping at row=test-1732014341354 ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,297 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1332 2024-11-19T11:06:13,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,333 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-19T11:06:13,337 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732014373297 2024-11-19T11:06:13,343 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732014352089} 2024-11-19T11:06:13,343 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-19T11:06:13,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:13,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:06:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014373389 (current time:1732014373389). 2024-11-19T11:06:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-19T11:06:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6cfa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:13,391 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:13,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:13,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:13,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf4022b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:13,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:13,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:13,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:13,394 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:13,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d57cc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:13,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:13,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:13,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:13,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:13,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:06:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-19T11:06:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:13,402 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:13,403 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:13,406 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:13,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741883_1059 (size=52) 2024-11-19T11:06:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:13,813 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:13,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60}] 2024-11-19T11:06:13,814 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:06:13,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-19T11:06:13,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:06:13,966 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 9eb809ed54235dbcfb549c070333cd60 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-19T11:06:13,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/75abb4beb3f141069f6ee47d05d477d3 is 158, key is trslm:hdfs://localhost:35831/backupUT\x00ns2:test-17320143413541/meta:log-roll-map/1732014370988/Put/seqid=0 2024-11-19T11:06:13,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741884_1060 (size=5914) 2024-11-19T11:06:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:14,388 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/75abb4beb3f141069f6ee47d05d477d3 2024-11-19T11:06:14,399 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/f43f2525623f4d4fbdd911f5d7c218f2 is 433, key is session:backup_1732014352089/session:context/1732014371449/Put/seqid=0 2024-11-19T11:06:14,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741885_1061 (size=5535) 2024-11-19T11:06:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:14,805 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/f43f2525623f4d4fbdd911f5d7c218f2 2024-11-19T11:06:14,812 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/75abb4beb3f141069f6ee47d05d477d3 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 2024-11-19T11:06:14,820 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3, entries=6, sequenceid=17, filesize=5.8 K 2024-11-19T11:06:14,821 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/f43f2525623f4d4fbdd911f5d7c218f2 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 2024-11-19T11:06:14,828 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2, entries=2, sequenceid=17, filesize=5.4 K 2024-11-19T11:06:14,829 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 863ms, sequenceid=17, compaction requested=false 2024-11-19T11:06:14,829 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 9eb809ed54235dbcfb549c070333cd60: 2024-11-19T11:06:14,829 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. for snapshot_backup_system completed. 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.' region-info for snapshot=snapshot_backup_system 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3] hfiles 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 for snapshot=snapshot_backup_system 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2] hfiles 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 for snapshot=snapshot_backup_system 2024-11-19T11:06:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 for snapshot=snapshot_backup_system 2024-11-19T11:06:14,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741886_1062 (size=173) 2024-11-19T11:06:15,236 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:06:15,236 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-19T11:06:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-19T11:06:15,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:06:15,237 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:06:15,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-19T11:06:15,241 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:15,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 in 1.4250 sec 2024-11-19T11:06:15,242 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:15,242 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:15,243 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-19T11:06:15,244 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-19T11:06:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741887_1063 (size=810) 2024-11-19T11:06:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:15,655 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:15,664 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:15,665 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_backup_system 2024-11-19T11:06:15,666 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:15,667 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-19T11:06:15,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2670 sec 2024-11-19T11:06:17,322 DEBUG [master/c28903dc2d16:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732014341354 2024-11-19T11:06:17,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741888_1064 (size=950) 2024-11-19T11:06:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-19T11:06:17,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-19T11:06:17,562 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732014373297 started at 1732014377562. 2024-11-19T11:06:17,564 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732014373297 has been started. 2024-11-19T11:06:17,569 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17320143413541, test-1732014341354] 2024-11-19T11:06:17,572 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732014343060for backupID backup_1732014373297 2024-11-19T11:06:17,573 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-19T11:06:17,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-19T11:06:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-19T11:06:17,574 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-19T11:06:17,574 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-19T11:06:17,574 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-19T11:06:17,574 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-19T11:06:17,575 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,575 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:17,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:06:17,606 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,607 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:06:17,607 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:06:17,607 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,607 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-19T11:06:17,607 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:17,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-19T11:06:17,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:17,608 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-19T11:06:17,609 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-19T11:06:17,609 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-19T11:06:17,609 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-19T11:06:17,609 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-19T11:06:17,609 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-19T11:06:17,609 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-19T11:06:17,609 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure (rolllog) in zk 2024-11-19T11:06:17,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,617 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:06:17,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:06:17,617 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,617 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-19T11:06:17,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:06:17,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:06:17,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:17,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:06:17,620 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-19T11:06:17,620 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@32b44cf7[Count = 0] remaining members to acquire global barrier 2024-11-19T11:06:17,620 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-19T11:06:17,620 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,627 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,628 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-19T11:06:17,628 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,628 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-19T11:06:17,628 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-19T11:06:17,628 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,628 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732014358014 highest: 1732014358034 on c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,628 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.meta:.meta(num 1732014357995) roll requested 2024-11-19T11:06:17,640 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta, exclude list is [], retry=0 2024-11-19T11:06:17,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:06:17,645 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta 2024-11-19T11:06:17,646 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:06:17,646 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta is not closed yet, will try archiving it next time 2024-11-19T11:06:17,646 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0:(num 1732014358014) roll requested 2024-11-19T11:06:17,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741857_1033 (size=93) 2024-11-19T11:06:17,647 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta not finished, retry = 0 2024-11-19T11:06:17,659 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646, exclude list is [], retry=0 2024-11-19T11:06:17,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:06:17,663 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 2024-11-19T11:06:17,664 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:06:17,664 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 is not closed yet, will try archiving it next time 2024-11-19T11:06:17,664 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1:(num 1732014358034) roll requested 2024-11-19T11:06:17,665 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 not finished, retry = 0 2024-11-19T11:06:17,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741858_1034 (size=1198) 2024-11-19T11:06:17,678 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664, exclude list is [], retry=0 2024-11-19T11:06:17,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:06:17,683 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 2024-11-19T11:06:17,683 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:06:17,683 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 is not closed yet, will try archiving it next time 2024-11-19T11:06:17,684 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014343060 2024-11-19T11:06:17,685 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 not finished, retry = 0 2024-11-19T11:06:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741859_1035 (size=30490) 2024-11-19T11:06:17,688 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-19T11:06:17,689 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732014377646 on c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:17,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:17,711 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-19T11:06:17,712 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'c28903dc2d16,43005,1732014332227' in zk 2024-11-19T11:06:17,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,722 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-19T11:06:17,722 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:06:17,722 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-19T11:06:17,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:06:17,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:06:17,723 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:06:17,723 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:06:17,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:17,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,725 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:06:17,725 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:17,726 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,726 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'c28903dc2d16,43005,1732014332227': 2024-11-19T11:06:17,726 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-19T11:06:17,726 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'c28903dc2d16,43005,1732014332227' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-19T11:06:17,726 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-19T11:06:17,726 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-19T11:06:17,726 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-19T11:06:17,727 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-19T11:06:17,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:06:17,733 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:06:17,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:06:17,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:06:17,733 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:06:17,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:06:17,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:17,734 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:06:17,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:17,735 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000002.908 2024-11-19T11:06:17,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,735 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000003.950 2024-11-19T11:06:17,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:06:17,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:17,736 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732014341354 in 416 msec 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:17,743 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:06:17,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:17,744 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:06:17,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-19T11:06:17,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-19T11:06:17,744 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:06:17,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:06:17,749 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta 2024-11-19T11:06:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-19T11:06:17,755 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {c28903dc2d16:43005=1732014343060} newestTimestamps: {c28903dc2d16:43005=1732014358034} 2024-11-19T11:06:17,758 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: c28903dc2d16%2C43005%2C1732014332227.meta.1732014336578.meta 2024-11-19T11:06:17,758 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta 2024-11-19T11:06:17,758 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 2024-11-19T11:06:17,758 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 2024-11-19T11:06:17,758 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 2024-11-19T11:06:17,758 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 2024-11-19T11:06:17,759 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta 2024-11-19T11:06:17,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741892_1068 (size=950) 2024-11-19T11:06:18,171 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732014341354 target: hdfs://localhost:35831/backupUT/backup_1732014373297/default/test-1732014341354 descriptor: 'test-1732014341354', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:06:18,171 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:06:18,174 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732014341354 2024-11-19T11:06:18,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741893_1069 (size=53) 2024-11-19T11:06:18,580 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732014341354 2024-11-19T11:06:18,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741894_1070 (size=364) 2024-11-19T11:06:18,994 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17320143413541 target: hdfs://localhost:35831/backupUT/backup_1732014373297/ns2/test-17320143413541 descriptor: 'ns2:test-17320143413541', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:06:18,994 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:06:18,996 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17320143413541 2024-11-19T11:06:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741895_1071 (size=50) 2024-11-19T11:06:19,402 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17320143413541 2024-11-19T11:06:19,411 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:35831/backupUT/.tmp/backup_1732014373297 from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014;hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 2024-11-19T11:06:19,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e53388b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:19,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:19,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:19,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:19,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:19,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:19,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca35071, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:19,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:19,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:19,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:19,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:19,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4869a38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:19,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:06:19,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is c28903dc2d16,39427,1732014330695 2024-11-19T11:06:19,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67b500c6 2024-11-19T11:06:19,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:06:19,418 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:06:19,423 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-19T11:06:19,423 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17320143413541 2024-11-19T11:06:19,424 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17320143413541,, stopping at row=ns2:test-17320143413541 ,, for max=2147483647 with caching=100 2024-11-19T11:06:19,424 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:19,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:19,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:19,426 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:19,428 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17320143413541: ns2:test-17320143413541; 2024-11-19T11:06:19,429 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732014341354 2024-11-19T11:06:19,429 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732014341354,, stopping at row=test-1732014341354 ,, for max=2147483647 with caching=100 2024-11-19T11:06:19,431 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732014341354: test-1732014341354; 2024-11-19T11:06:19,431 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-19T11:06:19,432 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/hbase-staging/partitions_98437b2d-cd97-42b7-bdf9-72acc9e04f70 2024-11-19T11:06:19,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741896_1072 (size=192) 2024-11-19T11:06:19,855 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:19,855 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:19,855 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,784 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-2727931505241933844.jar 2024-11-19T11:06:20,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,785 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,786 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,787 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,787 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,787 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:20,787 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:20,788 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:20,788 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:20,788 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:20,789 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:20,789 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:20,789 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:20,789 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:20,789 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:20,790 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:20,791 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,791 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,791 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,792 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,792 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,792 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:20,792 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:20,793 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17320143413541,default:test-1732014341354 2024-11-19T11:06:20,793 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:06:20,793 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:06:20,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:20,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:20,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:20,794 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:21,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-143343519909320410.jar 2024-11-19T11:06:21,691 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:21,715 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 for WAL files 2024-11-19T11:06:21,716 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 2024-11-19T11:06:21,716 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 for WAL files 2024-11-19T11:06:21,717 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 2024-11-19T11:06:21,810 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1970857028_0003_9a56cba7-18ae-414e-8392-4704f64f3644/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:21,811 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1970857028_0003_9a56cba7-18ae-414e-8392-4704f64f3644/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:21,898 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 for hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 (-9223372036854775808:9223372036854775807) length:30490 2024-11-19T11:06:21,919 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-19T11:06:21,919 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-19T11:06:21,961 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 for hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 (-9223372036854775808:9223372036854775807) length:1198 2024-11-19T11:06:21,965 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-19T11:06:21,965 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-19T11:06:22,054 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e42e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,054 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:22,054 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:22,055 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:22,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:22,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:22,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cd8fa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:22,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:22,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,057 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:22,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23526a59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,059 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:22,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:22,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:22,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:22,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17320143413541', row='row-t20', locateType=CURRENT is [region=ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:06:22,064 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-19T11:06:22,064 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:06:22,064 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,064 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:22,064 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-19T11:06:22,065 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: c28903dc2d16 2024-11-19T11:06:22,071 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/backupUT/.tmp/backup_1732014373297/_temporary/0/_temporary/attempt_local1970857028_0003_r_000000_0/ns2/test-17320143413541/f/6d517fe679984ef99779f7bfd80738a5 is 38, key is row-t20/f:q1/1732014371951/Put/seqid=0 2024-11-19T11:06:22,074 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [c28903dc2d16:43005] Specified favored nodes: [c28903dc2d16:43005] 2024-11-19T11:06:22,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741897_1073 (size=5275) 2024-11-19T11:06:22,511 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c92d513, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,511 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:22,512 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:22,513 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:22,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:22,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:22,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7ef109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:22,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:22,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:22,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e01aab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,516 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:22,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:22,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:22,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:22,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732014341354', row='row-test-1732014341354-1-0', locateType=CURRENT is [region=test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., hostname=c28903dc2d16,43005,1732014332227, seqNum=107] 2024-11-19T11:06:22,521 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-19T11:06:22,521 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:06:22,521 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,521 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:22,522 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732014341354-1-0] 2024-11-19T11:06:22,522 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: c28903dc2d16 2024-11-19T11:06:22,534 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6310019c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,535 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:22,535 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:22,536 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e170a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:22,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:22,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf92d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:22,539 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:22,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:22,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:22,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:22,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732014341354', row='row-test-1732014341354-3-0', locateType=CURRENT is [region=test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., hostname=c28903dc2d16,43005,1732014332227, seqNum=107] 2024-11-19T11:06:22,544 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-19T11:06:22,544 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:06:22,544 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,544 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:22,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:22,544 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732014341354-3-0] 2024-11-19T11:06:22,544 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: c28903dc2d16 2024-11-19T11:06:22,560 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/backupUT/.tmp/backup_1732014373297/_temporary/0/_temporary/attempt_local1970857028_0003_r_000001_0/default/test-1732014341354/f/3faa5771cd214b8184ca10ad2be0260b is 59, key is row-test-1732014341354-1-10/f:q1/1732014371522/Put/seqid=0 2024-11-19T11:06:22,562 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [c28903dc2d16:43005] Specified favored nodes: [c28903dc2d16:43005] 2024-11-19T11:06:22,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741898_1074 (size=10770) 2024-11-19T11:06:22,966 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/backupUT/.tmp/backup_1732014373297/_temporary/0/_temporary/attempt_local1970857028_0003_r_000001_0/default/test-1732014341354/mob/d9263fc5e88c49ee8fbf93e5575fecac is 63, key is row-test-1732014341354-3-100/mob:q1/1732014371910/Put/seqid=0 2024-11-19T11:06:22,968 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [c28903dc2d16:43005] Specified favored nodes: [c28903dc2d16:43005] 2024-11-19T11:06:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741899_1075 (size=11811) 2024-11-19T11:06:23,853 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:35831/backupUT 2024-11-19T11:06:23,853 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732014373297 2024-11-19T11:06:23,853 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-19T11:06:23,871 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:35831/backupUT/.tmp/backup_1732014373297, hdfs://localhost:35831/backupUT] 2024-11-19T11:06:23,941 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:24,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741900_1076 (size=5275) 2024-11-19T11:06:24,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741901_1077 (size=11811) 2024-11-19T11:06:24,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741902_1078 (size=10770) 2024-11-19T11:06:25,547 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-19T11:06:25,550 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732014373297 2024-11-19T11:06:25,551 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-19T11:06:25,551 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1731267521_0004 completed: true true 2024-11-19T11:06:25,552 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3824359 FILE: Number of bytes written=6210445 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231467 HDFS: Number of bytes written=2433185 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=249 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-19T11:06:25,552 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:35831/backupUT/.tmp/backup_1732014373297 to hdfs://localhost:35831/backupUT finished. 2024-11-19T11:06:25,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:test-1732014341354:', locateType=CURRENT is [region=backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4., hostname=c28903dc2d16,43005,1732014332227, seqNum=2] 2024-11-19T11:06:25,571 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732014373297 2024-11-19T11:06:25,575 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732014352089} 2024-11-19T11:06:25,575 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-19T11:06:25,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741903_1079 (size=384) 2024-11-19T11:06:25,981 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:35831/backupUT/backup_1732014373297/.backup.manifest 2024-11-19T11:06:25,981 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732014373297 finished: type=INCREMENTAL,tablelist=test-1732014341354;ns2:test-17320143413541,targetRootDir=hdfs://localhost:35831/backupUT,startts=1732014377562,completets=1732014385571,bytescopied=0 2024-11-19T11:06:25,981 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-19T11:06:25,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-19T11:06:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-19T11:06:25,991 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-19T11:06:25,994 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-19T11:06:25,998 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732014373297 completed. 2024-11-19T11:06:25,998 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-19T11:06:26,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,023 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:35831/backupUT/backup_1732014373297/.backup.manifest 2024-11-19T11:06:26,026 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732014373297/.backup.manifest 2024-11-19T11:06:26,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732014341354 from 'test-1732014341354', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732014341354', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:06:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732014341354 2024-11-19T11:06:26,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741904_1080 (size=908) 2024-11-19T11:06:26,045 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000003.950 2024-11-19T11:06:26,045 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000004.908 2024-11-19T11:06:26,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732014341354}] 2024-11-19T11:06:26,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE}] 2024-11-19T11:06:26,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE 2024-11-19T11:06:26,055 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=CLOSING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:06:26,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE because future has completed 2024-11-19T11:06:26,058 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:06:26,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:06:26,211 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,211 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:06:26,211 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 295075fcca1bb094ad7be004ade119ff, disabling compactions & flushes 2024-11-19T11:06:26,211 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,211 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,212 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. after waiting 0 ms 2024-11-19T11:06:26,212 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,217 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-19T11:06:26,218 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:06:26,218 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,218 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 295075fcca1bb094ad7be004ade119ff: Waiting for close lock at 1732014386211Running coprocessor pre-close hooks at 1732014386211Disabling compacts and flushes for region at 1732014386211Disabling writes for close at 1732014386212 (+1 ms)Writing region close event to WAL at 1732014386213 (+1 ms)Running coprocessor post-close hooks at 1732014386218 (+5 ms)Closed at 1732014386218 2024-11-19T11:06:26,218 WARN [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 295075fcca1bb094ad7be004ade119ff to self. 2024-11-19T11:06:26,221 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,222 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=CLOSED 2024-11-19T11:06:26,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:06:26,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T11:06:26,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 in 167 msec 2024-11-19T11:06:26,228 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE; state=CLOSED, location=c28903dc2d16,43005,1732014332227; forceNewPlan=false, retain=true 2024-11-19T11:06:26,379 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:06:26,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE because future has completed 2024-11-19T11:06:26,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:06:26,538 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,538 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 295075fcca1bb094ad7be004ade119ff, NAME => 'test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:06:26,538 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:06:26,538 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732014341354 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,539 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:06:26,539 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,539 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,540 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,541 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName f 2024-11-19T11:06:26,541 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:26,549 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950 2024-11-19T11:06:26,554 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 2024-11-19T11:06:26,555 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:06:26,555 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,556 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName f2 2024-11-19T11:06:26,556 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:26,557 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:06:26,557 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,558 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 295075fcca1bb094ad7be004ade119ff columnFamilyName mob 2024-11-19T11:06:26,558 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:26,565 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008 2024-11-19T11:06:26,570 DEBUG [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 2024-11-19T11:06:26,571 INFO [StoreOpener-295075fcca1bb094ad7be004ade119ff-1 {}] regionserver.HStore(327): Store=295075fcca1bb094ad7be004ade119ff/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:06:26,571 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,572 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,573 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,575 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,575 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,575 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732014341354 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-19T11:06:26,577 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,578 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 295075fcca1bb094ad7be004ade119ff; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9860592640, jitterRate=-0.08166074752807617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-19T11:06:26,578 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:26,581 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 295075fcca1bb094ad7be004ade119ff: Running coprocessor pre-open hook at 1732014386539Writing region info on filesystem at 1732014386539Initializing all the Stores at 1732014386540 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014386540Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014386540Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014386540Cleaning up temporary data from old regions at 1732014386575 (+35 ms)Running coprocessor post-open hooks at 1732014386578 (+3 ms)Region opened successfully at 1732014386581 (+3 ms) 2024-11-19T11:06:26,582 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., pid=46, masterSystemTime=1732014386534 2024-11-19T11:06:26,583 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,583 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:26,584 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=295075fcca1bb094ad7be004ade119ff, regionState=OPEN, openSeqNum=338, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:06:26,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:06:26,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-19T11:06:26,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 295075fcca1bb094ad7be004ade119ff, server=c28903dc2d16,43005,1732014332227 in 205 msec 2024-11-19T11:06:26,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-19T11:06:26,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732014341354, region=295075fcca1bb094ad7be004ade119ff, REOPEN/MOVE in 537 msec 2024-11-19T11:06:26,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T11:06:26,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732014341354 in 542 msec 2024-11-19T11:06:26,595 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732014341354 2024-11-19T11:06:26,600 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f3/c1df863c95e8403b9cac1416030240f9 2024-11-19T11:06:26,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732014341354 in 572 msec 2024-11-19T11:06:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T11:06:26,605 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732014341354 completed 2024-11-19T11:06:26,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,638 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,645 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-19T11:06:26,650 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732014386606 2024-11-19T11:06:26,652 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732014373297} 2024-11-19T11:06:26,652 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732014352089} 2024-11-19T11:06:26,652 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-19T11:06:26,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,706 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732014341354 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732014341354 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:06:26,710 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732014386606,startts=0,failedts=1732014386707,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732014341354 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-19T11:06:26,710 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-19T11:06:26,713 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-19T11:06:26,713 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-19T11:06:26,715 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-19T11:06:26,715 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-19T11:06:26,717 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-19T11:06:26,720 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732014386606 failed. 2024-11-19T11:06:26,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:26,742 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732014341354, ns2:test-17320143413541, ns3:test-17320143413542, ns4:test-17320143413543] 2024-11-19T11:06:26,743 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-19T11:06:26,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:06:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014386748 (current time:1732014386748). 2024-11-19T11:06:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-19T11:06:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450b0412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:26,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e84559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:26,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:26,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:26,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583072c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:26,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:26,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:26,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:26,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:26,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:06:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-19T11:06:26,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:26,761 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:26,763 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:26,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741905_1081 (size=52) 2024-11-19T11:06:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:26,892 DEBUG [master/c28903dc2d16:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732014341354 2024-11-19T11:06:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741906_1082 (size=950) 2024-11-19T11:06:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:27,171 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:27,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60}] 2024-11-19T11:06:27,173 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:06:27,304 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000004.908 2024-11-19T11:06:27,304 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/.tabledesc/.tableinfo.0000000005.950 2024-11-19T11:06:27,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732014341354 in 413 msec 2024-11-19T11:06:27,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2024-11-19T11:06:27,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:06:27,325 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9eb809ed54235dbcfb549c070333cd60 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-19T11:06:27,330 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/eae53c8151134230a72f7b1cfd56835b is 158, key is trslm:hdfs://localhost:35831/backupUT\x00ns2:test-17320143413541/meta:log-roll-map/1732014385558/Put/seqid=0 2024-11-19T11:06:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741907_1083 (size=5745) 2024-11-19T11:06:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:27,735 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/eae53c8151134230a72f7b1cfd56835b 2024-11-19T11:06:27,744 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/7e61520028204e358874a2563183612b is 1245, key is session:backup_1732014386606/session:context/1732014386716/Put/seqid=0 2024-11-19T11:06:27,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741908_1084 (size=6682) 2024-11-19T11:06:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:28,149 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/7e61520028204e358874a2563183612b 2024-11-19T11:06:28,156 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/eae53c8151134230a72f7b1cfd56835b as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b 2024-11-19T11:06:28,163 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b, entries=4, sequenceid=31, filesize=5.6 K 2024-11-19T11:06:28,165 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/7e61520028204e358874a2563183612b as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b 2024-11-19T11:06:28,172 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b, entries=3, sequenceid=31, filesize=6.5 K 2024-11-19T11:06:28,174 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 848ms, sequenceid=31, compaction requested=true 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9eb809ed54235dbcfb549c070333cd60: 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. for snapshot_backup_system completed. 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.' region-info for snapshot=snapshot_backup_system 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b] hfiles 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 for snapshot=snapshot_backup_system 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b for snapshot=snapshot_backup_system 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b] hfiles 2024-11-19T11:06:28,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 for snapshot=snapshot_backup_system 2024-11-19T11:06:28,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 for snapshot=snapshot_backup_system 2024-11-19T11:06:28,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b for snapshot=snapshot_backup_system 2024-11-19T11:06:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741909_1085 (size=251) 2024-11-19T11:06:28,584 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:06:28,584 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T11:06:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T11:06:28,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:06:28,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:06:28,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=47 2024-11-19T11:06:28,589 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:28,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 in 1.4150 sec 2024-11-19T11:06:28,590 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:28,590 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:28,590 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-19T11:06:28,591 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-19T11:06:28,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741910_1086 (size=888) 2024-11-19T11:06:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:29,002 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:29,012 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:29,013 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_backup_system 2024-11-19T11:06:29,015 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:29,015 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-19T11:06:29,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2570 sec 2024-11-19T11:06:29,695 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:06:30,353 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 649b89cbd07b01704a6e5b30a452786a, had cached 0 bytes from a total of 8413 2024-11-19T11:06:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-19T11:06:30,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-19T11:06:30,914 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732014386721 started at 1732014390914. 2024-11-19T11:06:30,920 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732014386721 has been started. 2024-11-19T11:06:30,926 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-19T11:06:30,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-19T11:06:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-19T11:06:30,927 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-19T11:06:30,927 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-19T11:06:30,927 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-19T11:06:30,927 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-19T11:06:30,928 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:30,928 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:31,064 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:06:31,065 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:06:31,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:06:31,066 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,066 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-19T11:06:31,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:31,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-19T11:06:31,071 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:31,071 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-19T11:06:31,071 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-19T11:06:31,072 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-19T11:06:31,072 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-19T11:06:31,072 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-19T11:06:31,072 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-19T11:06:31,072 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-19T11:06:31,072 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure (rolllog) in zk 2024-11-19T11:06:31,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,090 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:06:31,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:06:31,090 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,090 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-19T11:06:31,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:06:31,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:06:31,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:31,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:06:31,093 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-19T11:06:31,093 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1a8f34cf[Count = 0] remaining members to acquire global barrier 2024-11-19T11:06:31,093 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-19T11:06:31,093 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,100 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,100 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-19T11:06:31,100 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,100 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-19T11:06:31,101 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-19T11:06:31,101 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,101 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732014377646 highest: 1732014377664 on c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,101 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.meta:.meta(num 1732014377628) roll requested 2024-11-19T11:06:31,116 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta, exclude list is [], retry=0 2024-11-19T11:06:31,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:06:31,120 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta 2024-11-19T11:06:31,120 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:06:31,120 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta is not closed yet, will try archiving it next time 2024-11-19T11:06:31,120 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0:(num 1732014377646) roll requested 2024-11-19T11:06:31,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741889_1065 (size=1903) 2024-11-19T11:06:31,121 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta not finished, retry = 0 2024-11-19T11:06:31,133 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120, exclude list is [], retry=0 2024-11-19T11:06:31,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:06:31,137 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 2024-11-19T11:06:31,137 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:06:31,137 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 is not closed yet, will try archiving it next time 2024-11-19T11:06:31,137 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1:(num 1732014377664) roll requested 2024-11-19T11:06:31,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741890_1066 (size=93) 2024-11-19T11:06:31,139 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 not finished, retry = 0 2024-11-19T11:06:31,149 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137, exclude list is [], retry=0 2024-11-19T11:06:31,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:06:31,153 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137 2024-11-19T11:06:31,153 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:06:31,153 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 is not closed yet, will try archiving it next time 2024-11-19T11:06:31,153 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014358034 2024-11-19T11:06:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741891_1067 (size=4927) 2024-11-19T11:06:31,155 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 not finished, retry = 0 2024-11-19T11:06:31,161 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-19T11:06:31,161 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732014391120 on c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:31,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:31,176 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-19T11:06:31,176 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'c28903dc2d16,43005,1732014332227' in zk 2024-11-19T11:06:31,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,184 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-19T11:06:31,184 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:06:31,184 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-19T11:06:31,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:06:31,184 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:06:31,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:06:31,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:06:31,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:31,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,187 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:06:31,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:31,188 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'c28903dc2d16,43005,1732014332227': 2024-11-19T11:06:31,189 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-19T11:06:31,189 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'c28903dc2d16,43005,1732014332227' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-19T11:06:31,189 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-19T11:06:31,189 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-19T11:06:31,189 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-19T11:06:31,189 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-19T11:06:31,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:06:31,203 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,203 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,203 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:06:31,203 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,203 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:06:31,204 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:06:31,204 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:06:31,204 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:06:31,205 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:31,205 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:06:31,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:31,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,208 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:06:31,209 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:06:31,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:06:31,224 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,224 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:06:31,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:06:31,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:06:31,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:06:31,225 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:06:31,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-19T11:06:31,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-19T11:06:31,225 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:06:31,225 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:06:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-19T11:06:31,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-19T11:06:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014391236 (current time:1732014391236). 2024-11-19T11:06:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014391235_backup_system_bulk VERSION not specified, setting to 2 2024-11-19T11:06:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b661ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:31,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:31,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:31,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:31,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232f4875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:31,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:31,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:31,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:31,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:31,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48095298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:31,240 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014377646 2024-11-19T11:06:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:31,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:31,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:31,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:31,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:31,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-19T11:06:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-19T11:06:31,245 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:06:31,246 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:31,248 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:31,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741914_1090 (size=76) 2024-11-19T11:06:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:06:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:06:31,657 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:31,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e1b8b050c5202951d3c6975654ccd6c4}] 2024-11-19T11:06:31,657 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:06:31,750 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-19T11:06:31,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-19T11:06:31,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:06:31,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e1b8b050c5202951d3c6975654ccd6c4: 2024-11-19T11:06:31,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. for snapshot_1732014391235_backup_system_bulk completed. 2024-11-19T11:06:31,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.' region-info for snapshot=snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:31,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:31,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-19T11:06:31,810 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-19T11:06:31,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741915_1091 (size=61) 2024-11-19T11:06:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:06:32,218 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:06:32,219 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T11:06:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T11:06:32,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014391235_backup_system_bulk on region e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:06:32,221 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:06:32,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T11:06:32,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e1b8b050c5202951d3c6975654ccd6c4 in 567 msec 2024-11-19T11:06:32,228 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:32,229 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:32,230 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:32,231 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:32,232 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741916_1092 (size=702) 2024-11-19T11:06:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:06:32,649 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:32,657 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:32,658 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014391235_backup_system_bulk to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:32,659 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:32,659 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-19T11:06:32,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732014391235_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4160 sec 2024-11-19T11:06:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T11:06:33,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-19T11:06:33,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } 2024-11-19T11:06:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014393394 (current time:1732014393394). 2024-11-19T11:06:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014393393_ns4_test-17320143413543 VERSION not specified, setting to 2 2024-11-19T11:06:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@820452e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:33,398 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:33,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:33,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:33,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19db1419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:33,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:33,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:33,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:33,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:33,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6450b48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:33,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:33,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:33,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:33,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:33,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } 2024-11-19T11:06:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-19T11:06:33,409 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:06:33,409 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:33,412 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:33,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741917_1093 (size=86) 2024-11-19T11:06:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:06:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:06:33,819 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:33,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5e525c2317c76643ca583f53f5308f38}] 2024-11-19T11:06:33,820 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:06:33,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-19T11:06:33,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:06:33,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 5e525c2317c76643ca583f53f5308f38: 2024-11-19T11:06:33,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. for snapshot_1732014393393_ns4_test-17320143413543 completed. 2024-11-19T11:06:33,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38.' region-info for snapshot=snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:33,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:33,975 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-19T11:06:33,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741918_1094 (size=52) 2024-11-19T11:06:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:06:34,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:06:34,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T11:06:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T11:06:34,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014393393_ns4_test-17320143413543 on region 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:06:34,390 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:06:34,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T11:06:34,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 5e525c2317c76643ca583f53f5308f38 in 573 msec 2024-11-19T11:06:34,395 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:34,397 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:34,398 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:34,398 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:34,399 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:34,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741919_1095 (size=443) 2024-11-19T11:06:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:06:34,816 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:34,823 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:34,824 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014393393_ns4_test-17320143413543 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:34,825 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:34,825 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-19T11:06:34,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732014393393_ns4_test-17320143413543 table=ns4:test-17320143413543 type=FLUSH ttl=0 } in 1.4180 sec 2024-11-19T11:06:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T11:06:35,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17320143413543 completed 2024-11-19T11:06:35,554 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } 2024-11-19T11:06:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014395554 (current time:1732014395554). 2024-11-19T11:06:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014395553_default_test-1732014341354 VERSION not specified, setting to 2 2024-11-19T11:06:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2dfa10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:35,558 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:35,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:35,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:35,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c6a145, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:35,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:35,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:35,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:35,560 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:35,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140667d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:35,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:35,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:35,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:35,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:35,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } 2024-11-19T11:06:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-19T11:06:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T11:06:35,571 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:35,573 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:35,576 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:35,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741920_1096 (size=84) 2024-11-19T11:06:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T11:06:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T11:06:36,033 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:36,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff}] 2024-11-19T11:06:36,034 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:36,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-19T11:06:36,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:36,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 295075fcca1bb094ad7be004ade119ff: 2024-11-19T11:06:36,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. for snapshot_1732014395553_default_test-1732014341354 completed. 2024-11-19T11:06:36,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff.' region-info for snapshot=snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:36,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:36,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950] hfiles 2024-11-19T11:06:36,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 for snapshot=snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:36,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950 for snapshot=snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:36,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-19T11:06:36,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008] hfiles 2024-11-19T11:06:36,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 for snapshot=snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:36,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008 for snapshot=snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:36,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741921_1097 (size=224) 2024-11-19T11:06:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T11:06:36,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:06:36,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-19T11:06:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-19T11:06:36,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014395553_default_test-1732014341354 on region 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:36,604 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:36,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T11:06:36,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 295075fcca1bb094ad7be004ade119ff in 573 msec 2024-11-19T11:06:36,611 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:36,613 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:36,616 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-19T11:06:36,616 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-19T11:06:36,616 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:36,617 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-19T11:06:36,617 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:36,617 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-19T11:06:36,617 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:06:36,618 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff] hfiles 2024-11-19T11:06:36,618 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:36,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741922_1098 (size=168) 2024-11-19T11:06:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T11:06:36,892 INFO [master/c28903dc2d16:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T11:06:36,892 INFO [master/c28903dc2d16:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T11:06:37,025 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:37,025 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:37,026 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:37,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741923_1099 (size=1347) 2024-11-19T11:06:37,443 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:37,452 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:37,453 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014395553_default_test-1732014341354 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:37,454 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:37,454 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-19T11:06:37,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732014395553_default_test-1732014341354 table=test-1732014341354 type=FLUSH ttl=0 } in 1.8850 sec 2024-11-19T11:06:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T11:06:37,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732014341354 completed 2024-11-19T11:06:37,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } 2024-11-19T11:06:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014397726 (current time:1732014397726). 2024-11-19T11:06:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014397724_ns3_test-17320143413542 VERSION not specified, setting to 2 2024-11-19T11:06:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e656d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:37,730 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:37,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:37,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:37,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453bc224, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:37,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:37,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:37,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:37,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:37,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c7bf7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:37,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:37,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:37,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:37,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:37,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } 2024-11-19T11:06:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-19T11:06:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:06:37,743 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:37,744 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:37,747 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:37,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741924_1100 (size=86) 2024-11-19T11:06:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:06:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:06:38,155 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:38,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure af7e2bb446c6a1468013d39e5733df3e}] 2024-11-19T11:06:38,157 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:06:38,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-19T11:06:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:06:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for af7e2bb446c6a1468013d39e5733df3e: 2024-11-19T11:06:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. for snapshot_1732014397724_ns3_test-17320143413542 completed. 2024-11-19T11:06:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e.' region-info for snapshot=snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:38,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:38,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-19T11:06:38,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741925_1101 (size=52) 2024-11-19T11:06:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:06:38,405 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9eb809ed54235dbcfb549c070333cd60, had cached 0 bytes from a total of 28925 2024-11-19T11:06:38,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:06:38,724 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-19T11:06:38,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-19T11:06:38,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014397724_ns3_test-17320143413542 on region af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:06:38,725 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:06:38,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T11:06:38,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure af7e2bb446c6a1468013d39e5733df3e in 571 msec 2024-11-19T11:06:38,729 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:38,730 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:38,731 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:38,731 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:38,732 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:38,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741926_1102 (size=443) 2024-11-19T11:06:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:06:39,151 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:39,158 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:39,159 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014397724_ns3_test-17320143413542 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:39,160 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:39,160 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-19T11:06:39,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732014397724_ns3_test-17320143413542 table=ns3:test-17320143413542 type=FLUSH ttl=0 } in 1.4210 sec 2024-11-19T11:06:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-19T11:06:39,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17320143413542 completed 2024-11-19T11:06:39,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } 2024-11-19T11:06:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014399895 (current time:1732014399895). 2024-11-19T11:06:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:06:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732014399894_ns2_test-17320143413541 VERSION not specified, setting to 2 2024-11-19T11:06:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a25509f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:06:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:06:39,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:06:39,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:06:39,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:06:39,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473d9e77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:39,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:06:39,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:06:39,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:39,903 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:06:39,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f306d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:06:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:06:39,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:06:39,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:06:39,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:06:39,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:06:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:06:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:06:39,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:06:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:06:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } 2024-11-19T11:06:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-19T11:06:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T11:06:39,913 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:06:39,914 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:06:39,917 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:06:39,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741927_1103 (size=86) 2024-11-19T11:06:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T11:06:40,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T11:06:40,328 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:06:40,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a}] 2024-11-19T11:06:40,331 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:06:40,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-19T11:06:40,486 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:06:40,486 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 649b89cbd07b01704a6e5b30a452786a 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-19T11:06:40,495 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/.tmp/f/3b977d9db4c44b7eade8acb5f747b012 is 38, key is row-t20/f:q1/1732014371951/Put/seqid=0 2024-11-19T11:06:40,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741928_1104 (size=5134) 2024-11-19T11:06:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T11:06:40,904 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/.tmp/f/3b977d9db4c44b7eade8acb5f747b012 2024-11-19T11:06:40,919 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/.tmp/f/3b977d9db4c44b7eade8acb5f747b012 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/3b977d9db4c44b7eade8acb5f747b012 2024-11-19T11:06:40,926 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/3b977d9db4c44b7eade8acb5f747b012, entries=5, sequenceid=111, filesize=5.0 K 2024-11-19T11:06:40,927 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 649b89cbd07b01704a6e5b30a452786a in 441ms, sequenceid=111, compaction requested=false 2024-11-19T11:06:40,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 649b89cbd07b01704a6e5b30a452786a: 2024-11-19T11:06:40,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. for snapshot_1732014399894_ns2_test-17320143413541 completed. 2024-11-19T11:06:40,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a.' region-info for snapshot=snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:40,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:06:40,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/3b977d9db4c44b7eade8acb5f747b012] hfiles 2024-11-19T11:06:40,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 for snapshot=snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:40,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/3b977d9db4c44b7eade8acb5f747b012 for snapshot=snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:40,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741929_1105 (size=130) 2024-11-19T11:06:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T11:06:41,335 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:06:41,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-19T11:06:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-19T11:06:41,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732014399894_ns2_test-17320143413541 on region 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:06:41,338 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:06:41,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-19T11:06:41,345 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:06:41,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 649b89cbd07b01704a6e5b30a452786a in 1.0130 sec 2024-11-19T11:06:41,346 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:06:41,346 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:06:41,347 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:41,347 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:41,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741930_1106 (size=496) 2024-11-19T11:06:41,760 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:06:41,770 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:06:41,770 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_1732014399894_ns2_test-17320143413541 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:41,771 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:06:41,771 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-19T11:06:41,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732014399894_ns2_test-17320143413541 table=ns2:test-17320143413541 type=FLUSH ttl=0 } in 1.8610 sec 2024-11-19T11:06:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T11:06:42,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17320143413541 completed 2024-11-19T11:06:42,074 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732014386721 2024-11-19T11:06:42,074 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-19T11:06:42,074 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-19T11:06:42,074 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014386721_backup:system_bulk 2024-11-19T11:06:42,075 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014391235_backup_system_bulk to hdfs://localhost:35831/backupUT/backup_1732014386721/backup/system_bulk/ 2024-11-19T11:06:42,075 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:42,115 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:42,116 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014386721/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014386721/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:42,118 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:42,122 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014391235_backup_system_bulk to hdfs://localhost:35831/backupUT/backup_1732014386721/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:42,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741931_1107 (size=76) 2024-11-19T11:06:42,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741932_1108 (size=702) 2024-11-19T11:06:42,538 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:42,540 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:42,540 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-372922856465327010.jar 2024-11-19T11:06:43,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,406 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:43,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:43,407 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:43,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:43,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:43,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:43,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:43,408 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:43,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:43,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:43,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:43,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:43,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:43,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:43,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:43,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:43,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:43,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:43,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:43,447 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:43,470 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014391235_backup_system_bulk' hfile list 2024-11-19T11:06:43,547 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1428869586_0005_a7402357-26a2-42b7-a668-4d3d0c2781c6/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:43,547 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1428869586_0005_a7402357-26a2-42b7-a668-4d3d0c2781c6/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:44,588 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:44,590 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:44,600 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:44,600 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014391235_backup_system_bulk finished. 2024-11-19T11:06:44,600 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014386721_ns4:test-17320143413543 2024-11-19T11:06:44,600 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014393393_ns4_test-17320143413543 to hdfs://localhost:35831/backupUT/backup_1732014386721/ns4/test-17320143413543/ 2024-11-19T11:06:44,600 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:44,631 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:44,631 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014386721/ns4/test-17320143413543, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014386721/ns4/test-17320143413543/.hbase-snapshot/.tmp/snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:44,632 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:44,636 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014393393_ns4_test-17320143413543 to hdfs://localhost:35831/backupUT/backup_1732014386721/ns4/test-17320143413543/.hbase-snapshot/.tmp/snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:44,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741934_1110 (size=443) 2024-11-19T11:06:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741933_1109 (size=86) 2024-11-19T11:06:45,052 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,053 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,053 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-16181013876404678318.jar 2024-11-19T11:06:45,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,923 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,924 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:45,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:45,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:45,925 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:45,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:45,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:45,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:45,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:45,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:45,926 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:45,927 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:45,927 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:45,927 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:45,927 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:45,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:45,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:45,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:45,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:45,928 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:45,961 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:45,984 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014393393_ns4_test-17320143413543' hfile list 2024-11-19T11:06:46,057 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local412056745_0006_c6a55465-ff59-4175-8bb2-44593365ad84/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:46,057 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local412056745_0006_c6a55465-ff59-4175-8bb2-44593365ad84/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:47,091 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:47,093 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:47,102 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:47,102 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014393393_ns4_test-17320143413543 finished. 2024-11-19T11:06:47,102 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014386721_test-1732014341354 2024-11-19T11:06:47,102 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014395553_default_test-1732014341354 to hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/ 2024-11-19T11:06:47,102 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:47,130 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:47,130 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/.hbase-snapshot/.tmp/snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:47,132 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:47,139 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014395553_default_test-1732014341354 to hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/.hbase-snapshot/.tmp/snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741936_1112 (size=84) 2024-11-19T11:06:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741935_1111 (size=1347) 2024-11-19T11:06:47,552 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:47,552 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:47,552 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,417 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-4954179735568347186.jar 2024-11-19T11:06:48,418 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,418 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,418 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,419 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,419 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,419 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,419 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,419 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:48,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:48,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:48,420 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:48,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:48,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:48,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:48,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:48,421 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:48,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:48,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:48,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:48,422 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:48,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:48,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:48,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:48,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:48,423 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:48,424 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:48,460 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:48,484 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014395553_default_test-1732014341354' hfile list 2024-11-19T11:06:48,486 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=43.0 K 2024-11-19T11:06:48,570 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1852589679_0007_a7d018c8-12e1-4b8f-bf8a-b05b55623b81/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:48,570 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1852589679_0007_a7d018c8-12e1-4b8f-bf8a-b05b55623b81/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:48,635 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-19T11:06:48,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741937_1113 (size=13760) 2024-11-19T11:06:49,070 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008 output=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/3bc5d5500d8f420f9635501399f50008 2024-11-19T11:06:49,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-19T11:06:49,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741938_1114 (size=10796) 2024-11-19T11:06:49,502 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950 output=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/3a4b0c30a3534ca3824eb1970bbab950 2024-11-19T11:06:49,502 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-19T11:06:49,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741939_1115 (size=8413) 2024-11-19T11:06:49,935 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 output=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f/eb6e11f7dbd740978d1356b438bf56b3 2024-11-19T11:06:49,935 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-19T11:06:49,947 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff to hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/archive/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff" truncated to max limit (512 characters) 2024-11-19T11:06:49,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741940_1116 (size=5591) 2024-11-19T11:06:50,364 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/mobdir/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff output=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/archive/data/default/test-1732014341354/eb128a54dafb4c20e40fd43776b6aa46/mob/d41d8cd98f00b204e9800998ecf8427e20241119b56b148eb70b4bb5aa6678daf2d0f094_295075fcca1bb094ad7be004ade119ff 2024-11-19T11:06:50,364 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-11-19T11:06:50,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741941_1117 (size=5484) 2024-11-19T11:06:50,799 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 output=hdfs://localhost:35831/backupUT/backup_1732014386721/default/test-1732014341354/archive/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/mob/e69629895bf34199a73f0520391b1145 2024-11-19T11:06:50,799 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-19T11:06:51,614 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:51,617 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:51,627 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:51,628 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014395553_default_test-1732014341354 finished. 2024-11-19T11:06:51,628 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014386721_ns3:test-17320143413542 2024-11-19T11:06:51,628 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014397724_ns3_test-17320143413542 to hdfs://localhost:35831/backupUT/backup_1732014386721/ns3/test-17320143413542/ 2024-11-19T11:06:51,628 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:51,657 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:51,657 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014386721/ns3/test-17320143413542, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014386721/ns3/test-17320143413542/.hbase-snapshot/.tmp/snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:51,659 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:51,663 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014397724_ns3_test-17320143413542 to hdfs://localhost:35831/backupUT/backup_1732014386721/ns3/test-17320143413542/.hbase-snapshot/.tmp/snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:51,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741942_1118 (size=443) 2024-11-19T11:06:51,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741943_1119 (size=86) 2024-11-19T11:06:52,077 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,077 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,078 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-11554594873990754139.jar 2024-11-19T11:06:52,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:52,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:52,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:52,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:52,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:52,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:52,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:52,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:52,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:52,951 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:52,951 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:52,951 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:52,951 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:52,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:52,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:52,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:52,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:52,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:52,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:52,986 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:53,009 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014397724_ns3_test-17320143413542' hfile list 2024-11-19T11:06:53,081 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1502345173_0008_dccf55f2-fbab-4d16-b4f4-b0114b28741b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:53,081 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local1502345173_0008_dccf55f2-fbab-4d16-b4f4-b0114b28741b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:54,117 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:54,119 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:54,128 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:54,128 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014397724_ns3_test-17320143413542 finished. 2024-11-19T11:06:54,128 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732014386721_ns2:test-17320143413541 2024-11-19T11:06:54,129 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732014399894_ns2_test-17320143413541 to hdfs://localhost:35831/backupUT/backup_1732014386721/ns2/test-17320143413541/ 2024-11-19T11:06:54,129 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-19T11:06:54,156 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:35831, inputRoot=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442 2024-11-19T11:06:54,156 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1411486087_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:35831/backupUT/backup_1732014386721/ns2/test-17320143413541, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:35831/backupUT/backup_1732014386721/ns2/test-17320143413541/.hbase-snapshot/.tmp/snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:54,158 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-19T11:06:54,163 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_1732014399894_ns2_test-17320143413541 to hdfs://localhost:35831/backupUT/backup_1732014386721/ns2/test-17320143413541/.hbase-snapshot/.tmp/snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741945_1121 (size=496) 2024-11-19T11:06:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741944_1120 (size=86) 2024-11-19T11:06:54,574 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:54,574 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:54,574 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,440 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-16773142175005820414.jar 2024-11-19T11:06:55,440 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:06:55,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:06:55,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:06:55,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:06:55,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:06:55,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:06:55,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:06:55,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:06:55,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:06:55,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:06:55,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:06:55,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:06:55,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:55,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:55,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:55,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:55,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-19T11:06:55,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:55,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:06:55,478 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:06:55,503 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1732014399894_ns2_test-17320143413541' hfile list 2024-11-19T11:06:55,505 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=13.2 K 2024-11-19T11:06:55,577 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local82701562_0009_b9eb32bb-e4ef-4f1d-bc3d-fcb66cc1d939/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-19T11:06:55,577 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/mapreduce.cluster.local.dir/job_local82701562_0009_b9eb32bb-e4ef-4f1d-bc3d-fcb66cc1d939/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-19T11:06:55,642 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-19T11:06:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741946_1122 (size=8413) 2024-11-19T11:06:56,066 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 output=hdfs://localhost:35831/backupUT/backup_1732014386721/ns2/test-17320143413541/archive/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/b156ab1ca4e541409b66d857a83a7b86 2024-11-19T11:06:56,066 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-19T11:06:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741947_1123 (size=5134) 2024-11-19T11:06:56,498 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/3b977d9db4c44b7eade8acb5f747b012 output=hdfs://localhost:35831/backupUT/backup_1732014386721/ns2/test-17320143413541/archive/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/f/3b977d9db4c44b7eade8acb5f747b012 2024-11-19T11:06:56,498 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-19T11:06:56,617 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-19T11:06:56,619 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-19T11:06:56,630 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:56,630 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732014399894_ns2_test-17320143413541 finished. 2024-11-19T11:06:56,630 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-19T11:06:56,630 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17320143413543 2024-11-19T11:06:56,630 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732014341354 2024-11-19T11:06:56,630 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17320143413542 2024-11-19T11:06:56,631 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17320143413541 2024-11-19T11:06:56,641 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732014386721 2024-11-19T11:06:56,641 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-19T11:06:56,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741948_1124 (size=209) 2024-11-19T11:06:57,048 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:35831/backupUT/backup_1732014386721/.backup.manifest 2024-11-19T11:06:57,048 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732014386721 finished: type=FULL,tablelist=backup:system_bulk;ns4:test-17320143413543;test-1732014341354;ns3:test-17320143413542;ns2:test-17320143413541,targetRootDir=hdfs://localhost:35831/backupUT,startts=1732014390914,completets=1732014416641,bytescopied=0 2024-11-19T11:06:57,048 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-19T11:06:57,048 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:57,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014391235_backup_system_bulk" type: DISABLED 2024-11-19T11:06:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014391235_backup_system_bulk 2024-11-19T11:06:57,060 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014391235_backup_system_bulk for backup backup_1732014386721 succeeded. 2024-11-19T11:06:57,060 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:57,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014393393_ns4_test-17320143413543" type: DISABLED 2024-11-19T11:06:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014393393_ns4_test-17320143413543 2024-11-19T11:06:57,064 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014393393_ns4_test-17320143413543 for backup backup_1732014386721 succeeded. 2024-11-19T11:06:57,065 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:57,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014395553_default_test-1732014341354" type: DISABLED 2024-11-19T11:06:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014395553_default_test-1732014341354 2024-11-19T11:06:57,069 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014395553_default_test-1732014341354 for backup backup_1732014386721 succeeded. 2024-11-19T11:06:57,069 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:57,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014397724_ns3_test-17320143413542" type: DISABLED 2024-11-19T11:06:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014397724_ns3_test-17320143413542 2024-11-19T11:06:57,073 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014397724_ns3_test-17320143413542 for backup backup_1732014386721 succeeded. 2024-11-19T11:06:57,073 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:57,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732014399894_ns2_test-17320143413541" type: DISABLED 2024-11-19T11:06:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732014399894_ns2_test-17320143413541 2024-11-19T11:06:57,077 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732014399894_ns2_test-17320143413541 for backup backup_1732014386721 succeeded. 2024-11-19T11:06:57,078 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-19T11:06:57,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-19T11:06:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-19T11:06:57,088 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-19T11:06:57,091 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-19T11:06:57,098 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732014386721 completed. 2024-11-19T11:06:57,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:06:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:06:59,695 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T11:07:02,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,225 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-19T11:07:02,230 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732014422132 2024-11-19T11:07:02,232 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732014386721} 2024-11-19T11:07:02,232 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-19T11:07:02,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:02,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:07:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014422331 (current time:1732014422331). 2024-11-19T11:07:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:07:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-19T11:07:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e46859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:07:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:07:02,333 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:07:02,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:07:02,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:07:02,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f453d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:02,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:07:02,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:07:02,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:02,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:07:02,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b0eacd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:07:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:07:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:07:02,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:07:02,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:07:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:07:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:02,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:07:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:07:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:07:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-19T11:07:02,342 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:07:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:02,343 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:07:02,345 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:07:02,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741949_1125 (size=52) 2024-11-19T11:07:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:02,753 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-19T11:07:02,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60}] 2024-11-19T11:07:02,756 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:02,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43005 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-19T11:07:02,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:02,911 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9eb809ed54235dbcfb549c070333cd60 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-19T11:07:02,921 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/caddc2391d624b4b9b54225cf10c5d0f is 158, key is trslm:hdfs://localhost:35831/backupUT\x00ns2:test-17320143413541/meta:log-roll-map/1732014416636/Put/seqid=0 2024-11-19T11:07:02,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741950_1126 (size=6646) 2024-11-19T11:07:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:03,327 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/caddc2391d624b4b9b54225cf10c5d0f 2024-11-19T11:07:03,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/a4301272fa7f4d28923629313ec0c717 is 826, key is session:backup_1732014386721/session:context/1732014417090/Put/seqid=0 2024-11-19T11:07:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741951_1127 (size=5928) 2024-11-19T11:07:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:03,742 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/a4301272fa7f4d28923629313ec0c717 2024-11-19T11:07:03,748 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/caddc2391d624b4b9b54225cf10c5d0f as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f 2024-11-19T11:07:03,754 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f, entries=12, sequenceid=42, filesize=6.5 K 2024-11-19T11:07:03,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/a4301272fa7f4d28923629313ec0c717 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717 2024-11-19T11:07:03,761 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717, entries=2, sequenceid=42, filesize=5.8 K 2024-11-19T11:07:03,762 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 852ms, sequenceid=42, compaction requested=true 2024-11-19T11:07:03,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9eb809ed54235dbcfb549c070333cd60: 2024-11-19T11:07:03,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. for snapshot_backup_system completed. 2024-11-19T11:07:03,762 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.' region-info for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f] hfiles 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717] hfiles 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b for snapshot=snapshot_backup_system 2024-11-19T11:07:03,763 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717 for snapshot=snapshot_backup_system 2024-11-19T11:07:03,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741952_1128 (size=330) 2024-11-19T11:07:04,171 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:04,171 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-19T11:07:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-19T11:07:04,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:04,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:04,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T11:07:04,180 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-19T11:07:04,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9eb809ed54235dbcfb549c070333cd60 in 1.4230 sec 2024-11-19T11:07:04,181 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:07:04,181 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:07:04,182 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-19T11:07:04,183 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-19T11:07:04,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741953_1129 (size=967) 2024-11-19T11:07:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:04,597 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:07:04,609 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:07:04,610 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/snapshot_backup_system 2024-11-19T11:07:04,611 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:07:04,612 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-19T11:07:04,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2720 sec 2024-11-19T11:07:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-19T11:07:06,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-19T11:07:06,513 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732014422132 started at 1732014426513. 2024-11-19T11:07:06,517 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732014422132 has been started. 2024-11-19T11:07:06,525 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17320143413541, ns3:test-17320143413542, backup:system_bulk, ns4:test-17320143413543, test-1732014341354] 2024-11-19T11:07:06,529 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732014377664for backupID backup_1732014422132 2024-11-19T11:07:06,529 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-19T11:07:06,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-19T11:07:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-19T11:07:06,530 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-19T11:07:06,530 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-19T11:07:06,530 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-19T11:07:06,530 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-19T11:07:06,531 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,531 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:07:06,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:07:06,633 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,633 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:07:06,633 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:07:06,634 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,634 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-19T11:07:06,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-19T11:07:06,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,637 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-19T11:07:06,637 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-19T11:07:06,638 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-19T11:07:06,638 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-19T11:07:06,638 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-19T11:07:06,639 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-19T11:07:06,639 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-19T11:07:06,639 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-19T11:07:06,639 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-19T11:07:06,639 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure (rolllog) in zk 2024-11-19T11:07:06,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,656 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:07:06,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:07:06,657 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,657 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-19T11:07:06,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:07:06,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:07:06,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:07:06,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:07:06,663 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'c28903dc2d16,43005,1732014332227' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-19T11:07:06,663 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3f37f07c[Count = 0] remaining members to acquire global barrier 2024-11-19T11:07:06,663 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-19T11:07:06,663 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,674 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,675 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-19T11:07:06,675 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-19T11:07:06,675 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,675 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-19T11:07:06,675 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,676 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732014391120 highest: 1732014391137 on c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,676 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.meta:.meta(num 1732014391101) roll requested 2024-11-19T11:07:06,695 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014426676.meta, exclude list is [], retry=0 2024-11-19T11:07:06,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:07:06,699 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014426676.meta 2024-11-19T11:07:06,699 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:07:06,699 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta is not closed yet, will try archiving it next time 2024-11-19T11:07:06,699 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0:(num 1732014391120) roll requested 2024-11-19T11:07:06,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741911_1087 (size=93) 2024-11-19T11:07:06,700 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta not finished, retry = 0 2024-11-19T11:07:06,711 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014426699, exclude list is [], retry=0 2024-11-19T11:07:06,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:07:06,715 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014426699 2024-11-19T11:07:06,715 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:07:06,715 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 is not closed yet, will try archiving it next time 2024-11-19T11:07:06,715 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1:(num 1732014391137) roll requested 2024-11-19T11:07:06,715 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014358014 2024-11-19T11:07:06,716 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 not finished, retry = 0 2024-11-19T11:07:06,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741912_1088 (size=653) 2024-11-19T11:07:06,727 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014426715, exclude list is [], retry=0 2024-11-19T11:07:06,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37071,DS-ee598727-53e7-42c2-bf14-571c6ad4b0a1,DISK] 2024-11-19T11:07:06,731 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014426715 2024-11-19T11:07:06,731 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44235:44235)] 2024-11-19T11:07:06,731 DEBUG [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137 is not closed yet, will try archiving it next time 2024-11-19T11:07:06,731 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014377664 2024-11-19T11:07:06,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741913_1089 (size=4710) 2024-11-19T11:07:06,733 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137 not finished, retry = 0 2024-11-19T11:07:06,736 DEBUG [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-19T11:07:06,736 INFO [rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732014426699 on c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-19T11:07:06,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:06,753 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-19T11:07:06,753 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'c28903dc2d16,43005,1732014332227' in zk 2024-11-19T11:07:06,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,761 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-19T11:07:06,761 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:07:06,761 DEBUG [member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-19T11:07:06,761 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:07:06,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:07:06,762 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:07:06,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:07:06,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:07:06,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:07:06,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:07:06,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'c28903dc2d16,43005,1732014332227': 2024-11-19T11:07:06,766 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-19T11:07:06,766 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'c28903dc2d16,43005,1732014332227' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-19T11:07:06,766 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-19T11:07:06,766 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-19T11:07:06,766 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-19T11:07:06,766 INFO [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-19T11:07:06,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:07:06,779 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-19T11:07:06,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-19T11:07:06,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-19T11:07:06,780 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:07:06,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:07:06,781 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:07:06,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-19T11:07:06,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:07:06,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,783 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-19T11:07:06,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-19T11:07:06,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:07:06,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-19T11:07:06,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-19T11:07:06,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:06,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-19T11:07:06,794 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-19T11:07:06,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-19T11:07:06,794 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-19T11:07:06,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-19T11:07:06,794 DEBUG [(c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-19T11:07:06,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-19T11:07:06,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-19T11:07:06,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-19T11:07:06,795 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-19T11:07:06,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-19T11:07:06,802 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta 2024-11-19T11:07:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-19T11:07:06,807 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {c28903dc2d16:43005=1732014377664} newestTimestamps: {c28903dc2d16:43005=1732014391137} 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: c28903dc2d16%2C43005%2C1732014332227.meta.1732014336578.meta 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: c28903dc2d16%2C43005%2C1732014332227.meta.1732014377628.meta 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: c28903dc2d16%2C43005%2C1732014332227.meta.1732014426676.meta 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014426699 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137 2024-11-19T11:07:06,811 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014426715 2024-11-19T11:07:06,813 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.meta.1732014357995.meta 2024-11-19T11:07:06,813 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.meta.1732014391101.meta 2024-11-19T11:07:06,817 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 2024-11-19T11:07:06,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741957_1133 (size=640) 2024-11-19T11:07:07,228 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:35831/backupUT/backup_1732014422132/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:07:07,228 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:07:07,232 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-19T11:07:07,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741958_1134 (size=45) 2024-11-19T11:07:07,638 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-19T11:07:07,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741959_1135 (size=390) 2024-11-19T11:07:08,055 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17320143413543 target: hdfs://localhost:35831/backupUT/backup_1732014422132/ns4/test-17320143413543 descriptor: 'ns4:test-17320143413543', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:07:08,055 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:07:08,059 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17320143413543 2024-11-19T11:07:08,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741960_1136 (size=50) 2024-11-19T11:07:08,468 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17320143413543 2024-11-19T11:07:08,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741961_1137 (size=950) 2024-11-19T11:07:08,889 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732014341354 target: hdfs://localhost:35831/backupUT/backup_1732014422132/default/test-1732014341354 descriptor: 'test-1732014341354', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:07:08,889 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:07:08,894 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732014341354 2024-11-19T11:07:08,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741962_1138 (size=53) 2024-11-19T11:07:08,901 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732014341354 2024-11-19T11:07:08,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741963_1139 (size=390) 2024-11-19T11:07:09,312 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17320143413542 target: hdfs://localhost:35831/backupUT/backup_1732014422132/ns3/test-17320143413542 descriptor: 'ns3:test-17320143413542', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:07:09,312 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:07:09,315 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17320143413542 2024-11-19T11:07:09,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741964_1140 (size=50) 2024-11-19T11:07:09,725 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17320143413542 2024-11-19T11:07:09,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741965_1141 (size=364) 2024-11-19T11:07:10,145 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17320143413541 target: hdfs://localhost:35831/backupUT/backup_1732014422132/ns2/test-17320143413541 descriptor: 'ns2:test-17320143413541', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T11:07:10,145 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-19T11:07:10,149 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17320143413541 2024-11-19T11:07:10,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741966_1142 (size=50) 2024-11-19T11:07:10,557 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17320143413541 2024-11-19T11:07:10,570 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:35831/backupUT/.tmp/backup_1732014422132 from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120;hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1.1732014391137 2024-11-19T11:07:10,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a52add8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:10,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:07:10,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:07:10,572 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:07:10,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:07:10,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:07:10,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444fc620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:10,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:07:10,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:07:10,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:10,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:07:10,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c061324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:10,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T11:07:10,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is c28903dc2d16,39427,1732014330695 2024-11-19T11:07:10,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e260bb0 2024-11-19T11:07:10,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T11:07:10,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T11:07:10,579 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-19T11:07:10,579 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17320143413541 2024-11-19T11:07:10,579 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17320143413541,, stopping at row=ns2:test-17320143413541 ,, for max=2147483647 with caching=100 2024-11-19T11:07:10,579 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:07:10,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:07:10,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:07:10,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:07:10,583 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17320143413541: ns2:test-17320143413541; 2024-11-19T11:07:10,583 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17320143413542 2024-11-19T11:07:10,583 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17320143413542,, stopping at row=ns3:test-17320143413542 ,, for max=2147483647 with caching=100 2024-11-19T11:07:10,585 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17320143413542: ns3:test-17320143413542; 2024-11-19T11:07:10,585 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-19T11:07:10,585 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-19T11:07:10,586 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-19T11:07:10,586 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17320143413543 2024-11-19T11:07:10,586 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17320143413543,, stopping at row=ns4:test-17320143413543 ,, for max=2147483647 with caching=100 2024-11-19T11:07:10,588 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17320143413543: ns4:test-17320143413543; 2024-11-19T11:07:10,588 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732014341354 2024-11-19T11:07:10,588 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732014341354,, stopping at row=test-1732014341354 ,, for max=2147483647 with caching=100 2024-11-19T11:07:10,589 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732014341354: test-1732014341354; 2024-11-19T11:07:10,589 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-19T11:07:10,590 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/hbase-staging/partitions_7419c76e-01bc-4cc5-a121-11abbd9a1086 2024-11-19T11:07:10,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741967_1143 (size=324) 2024-11-19T11:07:10,998 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:10,999 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:10,999 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,539 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 295075fcca1bb094ad7be004ade119ff, had cached 0 bytes from a total of 38453 2024-11-19T11:07:11,860 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-3258528657968926424.jar 2024-11-19T11:07:11,861 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,861 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,861 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,861 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,862 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,862 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,862 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,862 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,862 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,863 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:07:11,863 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-19T11:07:11,863 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-19T11:07:11,863 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-19T11:07:11,863 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-19T11:07:11,864 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-19T11:07:11,864 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-19T11:07:11,864 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-19T11:07:11,864 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-19T11:07:11,864 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-19T11:07:11,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-19T11:07:11,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,865 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-19T11:07:11,866 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-19T11:07:11,866 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17320143413541,ns3:test-17320143413542,backup:system_bulk,ns4:test-17320143413543,default:test-1732014341354 2024-11-19T11:07:11,866 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:07:11,866 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:07:11,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:11,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:11,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:07:11,867 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-19T11:07:12,721 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop-12602323465653834024.jar 2024-11-19T11:07:12,731 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-19T11:07:12,756 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 for WAL files 2024-11-19T11:07:12,757 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1732014422132File hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 does not exist. java.io.FileNotFoundException: File hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T11:07:12,758 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732014422132,startts=1732014426513,failedts=1732014432758,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014391120 does not exist. 2024-11-19T11:07:12,758 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-19T11:07:12,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-11-19T11:07:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-19T11:07:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:07:12,773 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014432773"}]},"ts":"1732014432773"} 2024-11-19T11:07:12,775 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-19T11:07:12,775 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-19T11:07:12,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-19T11:07:12,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, UNASSIGN}] 2024-11-19T11:07:12,781 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, UNASSIGN 2024-11-19T11:07:12,782 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9eb809ed54235dbcfb549c070333cd60, regionState=CLOSING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:07:12,783 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39427 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=c28903dc2d16,43005,1732014332227, table=backup:system, region=9eb809ed54235dbcfb549c070333cd60. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T11:07:12,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, UNASSIGN because future has completed 2024-11-19T11:07:12,784 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T11:07:12,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:07:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:07:12,937 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:12,937 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T11:07:12,938 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 9eb809ed54235dbcfb549c070333cd60, disabling compactions & flushes 2024-11-19T11:07:12,938 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:12,938 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:12,938 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. after waiting 0 ms 2024-11-19T11:07:12,938 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:12,938 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 9eb809ed54235dbcfb549c070333cd60 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-19T11:07:12,945 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/dd632688cf044ac896ca0210c3b54026 is 103, key is rslogts:hdfs://localhost:35831/backupUT\x00c28903dc2d16:43005/meta:rs-log-ts/1732014426752/Put/seqid=0 2024-11-19T11:07:12,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741968_1144 (size=5338) 2024-11-19T11:07:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:07:13,353 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/dd632688cf044ac896ca0210c3b54026 2024-11-19T11:07:13,369 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/d3d6ae24acb54e2fa5b7ae1376c06f1d is 303, key is session:backup_1732014422132/session:context/1732014426515/Put/seqid=0 2024-11-19T11:07:13,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741969_1145 (size=5391) 2024-11-19T11:07:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:07:13,775 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/d3d6ae24acb54e2fa5b7ae1376c06f1d 2024-11-19T11:07:13,788 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/dd632688cf044ac896ca0210c3b54026 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/dd632688cf044ac896ca0210c3b54026 2024-11-19T11:07:13,796 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/dd632688cf044ac896ca0210c3b54026, entries=1, sequenceid=47, filesize=5.2 K 2024-11-19T11:07:13,797 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/d3d6ae24acb54e2fa5b7ae1376c06f1d as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/d3d6ae24acb54e2fa5b7ae1376c06f1d 2024-11-19T11:07:13,803 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/d3d6ae24acb54e2fa5b7ae1376c06f1d, entries=1, sequenceid=47, filesize=5.3 K 2024-11-19T11:07:13,803 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 865ms, sequenceid=47, compaction requested=true 2024-11-19T11:07:13,808 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-19T11:07:13,808 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:13,809 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:13,809 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 9eb809ed54235dbcfb549c070333cd60: Waiting for close lock at 1732014432937Running coprocessor pre-close hooks at 1732014432937Disabling compacts and flushes for region at 1732014432937Disabling writes for close at 1732014432938 (+1 ms)Obtaining lock to block concurrent updates at 1732014432938Preparing flush snapshotting stores in 9eb809ed54235dbcfb549c070333cd60 at 1732014432938Finished memstore snapshotting backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1732014432939 (+1 ms)Flushing stores of backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. at 1732014432939Flushing 9eb809ed54235dbcfb549c070333cd60/meta: creating writer at 1732014432940 (+1 ms)Flushing 9eb809ed54235dbcfb549c070333cd60/meta: appending metadata at 1732014432944 (+4 ms)Flushing 9eb809ed54235dbcfb549c070333cd60/meta: closing flushed file at 1732014432944Flushing 9eb809ed54235dbcfb549c070333cd60/session: creating writer at 1732014433366 (+422 ms)Flushing 9eb809ed54235dbcfb549c070333cd60/session: appending metadata at 1732014433369 (+3 ms)Flushing 9eb809ed54235dbcfb549c070333cd60/session: closing flushed file at 1732014433369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cb3f49d: reopening flushed file at 1732014433787 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@676ebb93: reopening flushed file at 1732014433797 (+10 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 865ms, sequenceid=47, compaction requested=true at 1732014433804 (+7 ms)Writing region close event to WAL at 1732014433805 (+1 ms)Running coprocessor post-close hooks at 1732014433808 (+3 ms)Closed at 1732014433809 (+1 ms) 2024-11-19T11:07:13,811 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:13,812 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9eb809ed54235dbcfb549c070333cd60, regionState=CLOSED 2024-11-19T11:07:13,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:07:13,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T11:07:13,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227 in 1.0310 sec 2024-11-19T11:07:13,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-19T11:07:13,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, UNASSIGN in 1.0360 sec 2024-11-19T11:07:13,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T11:07:13,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0440 sec 2024-11-19T11:07:13,823 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014433822"}]},"ts":"1732014433822"} 2024-11-19T11:07:13,824 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-19T11:07:13,824 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-19T11:07:13,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0580 sec 2024-11-19T11:07:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T11:07:13,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-19T11:07:13,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732014433927 2024-11-19T11:07:13,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:07:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732014433928 (current time:1732014433928). 2024-11-19T11:07:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-19T11:07:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1732014433927 VERSION not specified, setting to 2 2024-11-19T11:07:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5826bace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ClusterIdFetcher(90): Going to request c28903dc2d16,39427,-1 for getting cluster id 2024-11-19T11:07:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T11:07:13,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bc60509-da67-44f5-84a2-d7781db41c37' 2024-11-19T11:07:13,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T11:07:13,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bc60509-da67-44f5-84a2-d7781db41c37" 2024-11-19T11:07:13,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@649c8fc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:13,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c28903dc2d16,39427,-1] 2024-11-19T11:07:13,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T11:07:13,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:13,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T11:07:13,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b83baea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-19T11:07:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T11:07:13,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c28903dc2d16,43005,1732014332227, seqNum=-1] 2024-11-19T11:07:13,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T11:07:13,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T11:07:13,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427. 2024-11-19T11:07:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T11:07:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:13,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:07:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-19T11:07:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:07:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-19T11:07:13,937 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-19T11:07:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:07:13,937 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-19T11:07:13,939 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-19T11:07:13,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741970_1146 (size=81) 2024-11-19T11:07:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:07:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:07:14,348 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-19T11:07:14,351 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-19T11:07:14,351 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-19T11:07:14,351 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:07:14,356 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/dd632688cf044ac896ca0210c3b54026, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b] hfiles 2024-11-19T11:07:14,356 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 2024-11-19T11:07:14,356 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f 2024-11-19T11:07:14,356 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/dd632688cf044ac896ca0210c3b54026 2024-11-19T11:07:14,356 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b 2024-11-19T11:07:14,356 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:07:14,361 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/d3d6ae24acb54e2fa5b7ae1376c06f1d, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2] hfiles 2024-11-19T11:07:14,361 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 2024-11-19T11:07:14,361 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b 2024-11-19T11:07:14,361 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717 2024-11-19T11:07:14,361 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/d3d6ae24acb54e2fa5b7ae1376c06f1d 2024-11-19T11:07:14,361 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 2024-11-19T11:07:14,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741971_1147 (size=409) 2024-11-19T11:07:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:07:14,771 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 9eb809ed54235dbcfb549c070333cd60, NAME => 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-19T11:07:14,773 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-19T11:07:14,776 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-19T11:07:14,776 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1732014433927 2024-11-19T11:07:14,778 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732014433927 2024-11-19T11:07:14,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741972_1148 (size=1046) 2024-11-19T11:07:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:07:15,195 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-19T11:07:15,207 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-19T11:07:15,208 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732014433927 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1732014433927 2024-11-19T11:07:15,209 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-19T11:07:15,209 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-19T11:07:15,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732014433927 table=backup:system type=FLUSH ttl=0 } in 1.2740 sec 2024-11-19T11:07:15,353 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 649b89cbd07b01704a6e5b30a452786a, had cached 0 bytes from a total of 13547 2024-11-19T11:07:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T11:07:16,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-19T11:07:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732014422331 type: FLUSH version: 2 ttl: 0 ) 2024-11-19T11:07:16,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-19T11:07:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-19T11:07:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741973_1149 (size=635) 2024-11-19T11:07:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-19T11:07:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-19T11:07:16,528 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-19T11:07:16,528 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-19T11:07:16,531 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-19T11:07:16,537 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732014422331 type: FLUSH version: 2 ttl: 0 2024-11-19T11:07:16,538 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system 2024-11-19T11:07:16,541 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-19T11:07:16,541 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(229): region to restore: 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:16,545 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:07:16,548 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/dd632688cf044ac896ca0210c3b54026 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/dd632688cf044ac896ca0210c3b54026 2024-11-19T11:07:16,548 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:07:16,551 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/d3d6ae24acb54e2fa5b7ae1376c06f1d to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/d3d6ae24acb54e2fa5b7ae1376c06f1d 2024-11-19T11:07:16,552 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732014422331 type: FLUSH version: 2 ttl: 0 2024-11-19T11:07:16,553 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732014436553"}]},"ts":"9223372036854775807"} 2024-11-19T11:07:16,557 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T11:07:16,557 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9eb809ed54235dbcfb549c070333cd60, NAME => 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:07:16,558 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732014436554"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732014436554"}]},"ts":"1732014436554"} 2024-11-19T11:07:16,560 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T11:07:16,560 INFO [PEWorker-5 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-19T11:07:16,560 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => 9eb809ed54235dbcfb549c070333cd60, NAME => 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T11:07:16,560 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-19T11:07:16,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732014422331 type: FLUSH version: 2 ttl: 0 ) in 460 msec 2024-11-19T11:07:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-19T11:07:16,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732014433927 2024-11-19T11:07:16,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1732014433927" type: DISABLED 2024-11-19T11:07:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1732014433927 2024-11-19T11:07:16,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-11-19T11:07:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-19T11:07:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:07:16,766 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-19T11:07:16,767 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014436767"}]},"ts":"1732014436767"} 2024-11-19T11:07:16,768 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-19T11:07:16,769 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-19T11:07:16,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN}] 2024-11-19T11:07:16,771 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN 2024-11-19T11:07:16,771 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-19T11:07:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:07:16,923 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=9eb809ed54235dbcfb549c070333cd60, regionState=OPENING, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:07:16,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN because future has completed 2024-11-19T11:07:16,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227}] 2024-11-19T11:07:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:07:17,091 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,092 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => 9eb809ed54235dbcfb549c070333cd60, NAME => 'backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.', STARTKEY => '', ENDKEY => ''} 2024-11-19T11:07:17,092 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-19T11:07:17,093 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,093 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T11:07:17,093 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,093 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,097 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,099 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eb809ed54235dbcfb549c070333cd60 columnFamilyName meta 2024-11-19T11:07:17,099 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:07:17,108 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 2024-11-19T11:07:17,114 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f 2024-11-19T11:07:17,120 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b 2024-11-19T11:07:17,120 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(327): Store=9eb809ed54235dbcfb549c070333cd60/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:07:17,120 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,121 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9eb809ed54235dbcfb549c070333cd60 columnFamilyName session 2024-11-19T11:07:17,121 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T11:07:17,128 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 2024-11-19T11:07:17,132 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b 2024-11-19T11:07:17,136 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717 2024-11-19T11:07:17,140 DEBUG [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 2024-11-19T11:07:17,140 INFO [StoreOpener-9eb809ed54235dbcfb549c070333cd60-1 {}] regionserver.HStore(327): Store=9eb809ed54235dbcfb549c070333cd60/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T11:07:17,140 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,141 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,142 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,143 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,143 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,143 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-19T11:07:17,144 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,145 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened 9eb809ed54235dbcfb549c070333cd60; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10766287520, jitterRate=0.002688661217689514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-19T11:07:17,145 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,146 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for 9eb809ed54235dbcfb549c070333cd60: Running coprocessor pre-open hook at 1732014437093Writing region info on filesystem at 1732014437093Initializing all the Stores at 1732014437096 (+3 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014437096Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732014437096Cleaning up temporary data from old regions at 1732014437143 (+47 ms)Running coprocessor post-open hooks at 1732014437145 (+2 ms)Region opened successfully at 1732014437146 (+1 ms) 2024-11-19T11:07:17,147 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., pid=70, masterSystemTime=1732014437085 2024-11-19T11:07:17,148 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 9eb809ed54235dbcfb549c070333cd60:meta, priority=-2147483648, current under compaction store size is 1 2024-11-19T11:07:17,149 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:07:17,149 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T11:07:17,149 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 9eb809ed54235dbcfb549c070333cd60:session, priority=-2147483648, current under compaction store size is 2 2024-11-19T11:07:17,149 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:07:17,149 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T11:07:17,152 DEBUG [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,152 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T11:07:17,152 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T11:07:17,152 INFO [RS_OPEN_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,152 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=9eb809ed54235dbcfb549c070333cd60, regionState=OPEN, openSeqNum=51, regionLocation=c28903dc2d16,43005,1732014332227 2024-11-19T11:07:17,153 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.HStore(1541): 9eb809ed54235dbcfb549c070333cd60/meta is initiating minor compaction (all files) 2024-11-19T11:07:17,153 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.HStore(1541): 9eb809ed54235dbcfb549c070333cd60/session is initiating minor compaction (all files) 2024-11-19T11:07:17,154 INFO [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9eb809ed54235dbcfb549c070333cd60/meta in backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,154 INFO [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9eb809ed54235dbcfb549c070333cd60/session in backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,154 INFO [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp, totalSize=17.9 K 2024-11-19T11:07:17,154 INFO [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717] into tmpdir=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp, totalSize=22.7 K 2024-11-19T11:07:17,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227 because future has completed 2024-11-19T11:07:17,155 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b21082dcb0543e5bc0021ac30d34c67, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732014355613 2024-11-19T11:07:17,155 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75abb4beb3f141069f6ee47d05d477d3, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732014358088 2024-11-19T11:07:17,156 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae53c8151134230a72f7b1cfd56835b, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732014377710 2024-11-19T11:07:17,156 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] compactions.Compactor(225): Compacting f43f2525623f4d4fbdd911f5d7c218f2, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732014371449 2024-11-19T11:07:17,156 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] compactions.Compactor(225): Compacting caddc2391d624b4b9b54225cf10c5d0f, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732014391175 2024-11-19T11:07:17,156 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e61520028204e358874a2563183612b, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732014385993 2024-11-19T11:07:17,156 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] compactions.Compactor(225): Compacting a4301272fa7f4d28923629313ec0c717, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732014417090 2024-11-19T11:07:17,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-19T11:07:17,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 9eb809ed54235dbcfb549c070333cd60, server=c28903dc2d16,43005,1732014332227 in 225 msec 2024-11-19T11:07:17,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T11:07:17,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9eb809ed54235dbcfb549c070333cd60, ASSIGN in 389 msec 2024-11-19T11:07:17,160 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732014437160"}]},"ts":"1732014437160"} 2024-11-19T11:07:17,162 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-19T11:07:17,162 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-19T11:07:17,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 403 msec 2024-11-19T11:07:17,180 INFO [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9eb809ed54235dbcfb549c070333cd60#meta#compaction#16 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:07:17,180 INFO [RS:0;c28903dc2d16:43005-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9eb809ed54235dbcfb549c070333cd60#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T11:07:17,181 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/5edfbaa1733343f6bd5ad82f13b14a65 is 1245, key is session:backup_1732014386606/session:context/1732014386716/Put/seqid=0 2024-11-19T11:07:17,181 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/fde6ed7d750a43b6a94e65bf11f6e0d3 is 158, key is trslm:hdfs://localhost:35831/backupUT\x00ns2:test-17320143413541/meta:log-roll-map/1732014416636/Put/seqid=0 2024-11-19T11:07:17,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741975_1151 (size=8073) 2024-11-19T11:07:17,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741974_1150 (size=6749) 2024-11-19T11:07:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T11:07:17,393 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-19T11:07:17,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-19T11:07:17,393 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-19T11:07:17,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-19T11:07:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39427 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-19T11:07:17,404 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-19T11:07:17,404 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-19T11:07:17,406 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-19T11:07:17,410 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732014422132 failed. 2024-11-19T11:07:17,410 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:07:17,410 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:07:17,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:17,410 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:17,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:07:17,427 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 610) Potentially hanging thread: PacketResponder: BP-264477037-172.17.0.2-1732014326384:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/fde6ed7d750a43b6a94e65bf11f6e0d3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-264477037-172.17.0.2-1732014326384:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-705829449_20 at /127.0.0.1:33836 [Receiving block BP-264477037-172.17.0.2-1732014326384:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: rs(c28903dc2d16,43005,1732014332227)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/5edfbaa1733343f6bd5ad82f13b14a65 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: 'c28903dc2d16,43005,1732014332227' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-264477037-172.17.0.2-1732014326384:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (c28903dc2d16,39427,1732014330695)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1481076560_20 at /127.0.0.1:52048 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-705829449_20 at /127.0.0.1:33824 [Receiving block BP-264477037-172.17.0.2-1732014326384:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-705829449_20 at /127.0.0.1:33840 [Receiving block BP-264477037-172.17.0.2-1732014326384:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c28903dc2d16:43005-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/c28903dc2d16:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=641 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=96 (was 236), ProcessCount=9 (was 9), AvailableMemoryMB=14009 (was 14752) 2024-11-19T11:07:17,430 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-11-19T11:07:17,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T11:07:17,437 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:07:17,437 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:07:17,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:17,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:17,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:07:17,437 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T11:07:17,437 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1132182242, stopped=false 2024-11-19T11:07:17,437 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c28903dc2d16,39427,1732014330695 2024-11-19T11:07:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-19T11:07:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-19T11:07:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:07:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:07:17,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T11:07:17,498 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T11:07:17,498 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:07:17,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-19T11:07:17,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-19T11:07:17,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:17,499 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c28903dc2d16,43005,1732014332227' ***** 2024-11-19T11:07:17,499 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T11:07:17,499 INFO [RS:0;c28903dc2d16:43005 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-19T11:07:17,503 INFO [RS:0;c28903dc2d16:43005 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@7cd2e4d8{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-19T11:07:17,505 INFO [RS:0;c28903dc2d16:43005 {}] server.AbstractConnector(383): Stopped ServerConnector@91e124c{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-19T11:07:17,505 INFO [RS:0;c28903dc2d16:43005 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:17,505 INFO [RS:0;c28903dc2d16:43005 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3b20ef39{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-19T11:07:17,505 INFO [RS:0;c28903dc2d16:43005 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@340a8627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:17,507 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T11:07:17,507 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T11:07:17,507 INFO [RS:0;c28903dc2d16:43005 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T11:07:17,507 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for 5e525c2317c76643ca583f53f5308f38 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for 649b89cbd07b01704a6e5b30a452786a 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for 9eb809ed54235dbcfb549c070333cd60 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for 295075fcca1bb094ad7be004ade119ff 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for af7e2bb446c6a1468013d39e5733df3e 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(959): stopping server c28903dc2d16,43005,1732014332227 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T11:07:17,508 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5e525c2317c76643ca583f53f5308f38, disabling compactions & flushes 2024-11-19T11:07:17,508 INFO [RS:0;c28903dc2d16:43005 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c28903dc2d16:43005. 2024-11-19T11:07:17,508 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:07:17,508 DEBUG [RS:0;c28903dc2d16:43005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:07:17,508 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:07:17,508 DEBUG [RS:0;c28903dc2d16:43005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:17,508 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. after waiting 0 ms 2024-11-19T11:07:17,508 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:07:17,509 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T11:07:17,512 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns4/test-17320143413543/5e525c2317c76643ca583f53f5308f38/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:17,513 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5e525c2317c76643ca583f53f5308f38: Waiting for close lock at 1732014437508Running coprocessor pre-close hooks at 1732014437508Disabling compacts and flushes for region at 1732014437508Disabling writes for close at 1732014437508Writing region close event to WAL at 1732014437509 (+1 ms)Running coprocessor post-close hooks at 1732014437513 (+4 ms)Closed at 1732014437513 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17320143413543,,1732014348673.5e525c2317c76643ca583f53f5308f38. 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 649b89cbd07b01704a6e5b30a452786a, disabling compactions & flushes 2024-11-19T11:07:17,513 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. after waiting 0 ms 2024-11-19T11:07:17,513 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:07:17,517 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns2/test-17320143413541/649b89cbd07b01704a6e5b30a452786a/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-19T11:07:17,518 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:17,518 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:07:17,518 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 649b89cbd07b01704a6e5b30a452786a: Waiting for close lock at 1732014437513Running coprocessor pre-close hooks at 1732014437513Disabling compacts and flushes for region at 1732014437513Disabling writes for close at 1732014437513Writing region close event to WAL at 1732014437514 (+1 ms)Running coprocessor post-close hooks at 1732014437518 (+4 ms)Closed at 1732014437518 2024-11-19T11:07:17,518 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a. 2024-11-19T11:07:17,518 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9eb809ed54235dbcfb549c070333cd60, disabling compactions & flushes 2024-11-19T11:07:17,518 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,564 INFO [regionserver/c28903dc2d16:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T11:07:17,564 INFO [regionserver/c28903dc2d16:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T11:07:17,583 INFO [regionserver/c28903dc2d16:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T11:07:17,597 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/meta/fde6ed7d750a43b6a94e65bf11f6e0d3 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/fde6ed7d750a43b6a94e65bf11f6e0d3 2024-11-19T11:07:17,597 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/5edfbaa1733343f6bd5ad82f13b14a65 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/5edfbaa1733343f6bd5ad82f13b14a65 2024-11-19T11:07:17,613 INFO [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9eb809ed54235dbcfb549c070333cd60/meta of 9eb809ed54235dbcfb549c070333cd60 into fde6ed7d750a43b6a94e65bf11f6e0d3(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:07:17,613 INFO [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9eb809ed54235dbcfb549c070333cd60/session of 9eb809ed54235dbcfb549c070333cd60 into 5edfbaa1733343f6bd5ad82f13b14a65(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T11:07:17,613 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9eb809ed54235dbcfb549c070333cd60: 2024-11-19T11:07:17,613 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9eb809ed54235dbcfb549c070333cd60: 2024-11-19T11:07:17,613 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,613 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,613 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. after waiting 0 ms 2024-11-19T11:07:17,613 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:17,613 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9eb809ed54235dbcfb549c070333cd60 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-19T11:07:17,615 INFO [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., storeName=9eb809ed54235dbcfb549c070333cd60/meta, priority=13, startTime=1732014437148; duration=0sec 2024-11-19T11:07:17,615 INFO [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., storeName=9eb809ed54235dbcfb549c070333cd60/session, priority=12, startTime=1732014437149; duration=0sec 2024-11-19T11:07:17,615 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:07:17,615 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T11:07:17,615 DEBUG [RS:0;c28903dc2d16:43005-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9eb809ed54235dbcfb549c070333cd60:meta 2024-11-19T11:07:17,615 DEBUG [RS:0;c28903dc2d16:43005-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9eb809ed54235dbcfb549c070333cd60:session 2024-11-19T11:07:17,615 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T11:07:17,615 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T11:07:17,616 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T11:07:17,616 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-19T11:07:17,616 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1325): Online Regions={9eb809ed54235dbcfb549c070333cd60=backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., 1588230740=hbase:meta,,1.1588230740, 295075fcca1bb094ad7be004ade119ff=test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., af7e2bb446c6a1468013d39e5733df3e=ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e., e1b8b050c5202951d3c6975654ccd6c4=backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4.} 2024-11-19T11:07:17,616 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T11:07:17,616 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T11:07:17,616 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T11:07:17,616 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T11:07:17,616 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T11:07:17,616 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-19T11:07:17,616 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 295075fcca1bb094ad7be004ade119ff, 9eb809ed54235dbcfb549c070333cd60, af7e2bb446c6a1468013d39e5733df3e, e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:07:17,618 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/83c1afbca67d4216955d814afe81ff21 is 554, key is session:backup_1732014422132/session:context/1732014437405/Put/seqid=0 2024-11-19T11:07:17,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741976_1152 (size=5655) 2024-11-19T11:07:17,637 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/info/6d8b333fcf85414dbd2ca1bd30a7f6dd is 159, key is ns2:test-17320143413541,,1732014344196.649b89cbd07b01704a6e5b30a452786a./info:regioninfo/1732014345375/Put/seqid=0 2024-11-19T11:07:17,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741977_1153 (size=14126) 2024-11-19T11:07:17,817 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 295075fcca1bb094ad7be004ade119ff, 9eb809ed54235dbcfb549c070333cd60, af7e2bb446c6a1468013d39e5733df3e, e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:07:18,017 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 295075fcca1bb094ad7be004ade119ff, 9eb809ed54235dbcfb549c070333cd60, af7e2bb446c6a1468013d39e5733df3e, e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:07:18,024 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/83c1afbca67d4216955d814afe81ff21 2024-11-19T11:07:18,032 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/.tmp/session/83c1afbca67d4216955d814afe81ff21 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/83c1afbca67d4216955d814afe81ff21 2024-11-19T11:07:18,038 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/83c1afbca67d4216955d814afe81ff21, entries=2, sequenceid=57, filesize=5.5 K 2024-11-19T11:07:18,039 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 425ms, sequenceid=57, compaction requested=false 2024-11-19T11:07:18,039 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f] to archive 2024-11-19T11:07:18,040 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:07:18,041 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/info/6d8b333fcf85414dbd2ca1bd30a7f6dd 2024-11-19T11:07:18,044 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/75abb4beb3f141069f6ee47d05d477d3 2024-11-19T11:07:18,045 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/eae53c8151134230a72f7b1cfd56835b 2024-11-19T11:07:18,047 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/meta/caddc2391d624b4b9b54225cf10c5d0f 2024-11-19T11:07:18,053 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c28903dc2d16:39427 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-19T11:07:18,054 WARN [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [75abb4beb3f141069f6ee47d05d477d3=5914, eae53c8151134230a72f7b1cfd56835b=5745, caddc2391d624b4b9b54225cf10c5d0f=6646] 2024-11-19T11:07:18,054 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717] to archive 2024-11-19T11:07:18,055 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T11:07:18,057 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7b21082dcb0543e5bc0021ac30d34c67 2024-11-19T11:07:18,058 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/f43f2525623f4d4fbdd911f5d7c218f2 2024-11-19T11:07:18,060 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/7e61520028204e358874a2563183612b 2024-11-19T11:07:18,061 DEBUG [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717 to hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/archive/data/backup/system/9eb809ed54235dbcfb549c070333cd60/session/a4301272fa7f4d28923629313ec0c717 2024-11-19T11:07:18,061 WARN [StoreCloser-backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7b21082dcb0543e5bc0021ac30d34c67=5049, f43f2525623f4d4fbdd911f5d7c218f2=5535, 7e61520028204e358874a2563183612b=6682, a4301272fa7f4d28923629313ec0c717=5928] 2024-11-19T11:07:18,063 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/ns/e79267a41d41471d9ed8927c2563f719 is 43, key is default/ns:d/1732014336787/Put/seqid=0 2024-11-19T11:07:18,065 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system/9eb809ed54235dbcfb549c070333cd60/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-19T11:07:18,065 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:18,065 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:18,066 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9eb809ed54235dbcfb549c070333cd60: Waiting for close lock at 1732014437518Running coprocessor pre-close hooks at 1732014437518Disabling compacts and flushes for region at 1732014437518Disabling writes for close at 1732014437613 (+95 ms)Obtaining lock to block concurrent updates at 1732014437613Preparing flush snapshotting stores in 9eb809ed54235dbcfb549c070333cd60 at 1732014437613Finished memstore snapshotting backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1732014437613Flushing stores of backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. at 1732014437614 (+1 ms)Flushing 9eb809ed54235dbcfb549c070333cd60/session: creating writer at 1732014437614Flushing 9eb809ed54235dbcfb549c070333cd60/session: appending metadata at 1732014437617 (+3 ms)Flushing 9eb809ed54235dbcfb549c070333cd60/session: closing flushed file at 1732014437618 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fdb09a: reopening flushed file at 1732014438031 (+413 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 9eb809ed54235dbcfb549c070333cd60 in 425ms, sequenceid=57, compaction requested=false at 1732014438039 (+8 ms)Writing region close event to WAL at 1732014438061 (+22 ms)Running coprocessor post-close hooks at 1732014438065 (+4 ms)Closed at 1732014438065 2024-11-19T11:07:18,066 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732014352252.9eb809ed54235dbcfb549c070333cd60. 2024-11-19T11:07:18,066 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 295075fcca1bb094ad7be004ade119ff, disabling compactions & flushes 2024-11-19T11:07:18,066 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:07:18,066 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:07:18,066 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. after waiting 0 ms 2024-11-19T11:07:18,066 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:07:18,066 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 295075fcca1bb094ad7be004ade119ff 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-19T11:07:18,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741978_1154 (size=5345) 2024-11-19T11:07:18,079 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f2/ea47b365a49947c0aa89ffa6e753dc1c is 58, key is row-test-1732014341354-2-0/f2:q1/1732014417122/Put/seqid=0 2024-11-19T11:07:18,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741979_1155 (size=5464) 2024-11-19T11:07:18,217 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 295075fcca1bb094ad7be004ade119ff, af7e2bb446c6a1468013d39e5733df3e, e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:07:18,417 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 295075fcca1bb094ad7be004ade119ff, af7e2bb446c6a1468013d39e5733df3e, e1b8b050c5202951d3c6975654ccd6c4 2024-11-19T11:07:18,468 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/ns/e79267a41d41471d9ed8927c2563f719 2024-11-19T11:07:18,483 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f2/ea47b365a49947c0aa89ffa6e753dc1c 2024-11-19T11:07:18,490 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/.tmp/f2/ea47b365a49947c0aa89ffa6e753dc1c as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f2/ea47b365a49947c0aa89ffa6e753dc1c 2024-11-19T11:07:18,495 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/f2/ea47b365a49947c0aa89ffa6e753dc1c, entries=7, sequenceid=348, filesize=5.3 K 2024-11-19T11:07:18,496 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 295075fcca1bb094ad7be004ade119ff in 430ms, sequenceid=348, compaction requested=false 2024-11-19T11:07:18,501 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/default/test-1732014341354/295075fcca1bb094ad7be004ade119ff/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-19T11:07:18,501 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:18,501 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/table/ceaadff730324d4e92d1d84705231869 is 59, key is ns2:test-17320143413541/table:state/1732014345391/Put/seqid=0 2024-11-19T11:07:18,501 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:07:18,501 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 295075fcca1bb094ad7be004ade119ff: Waiting for close lock at 1732014438066Running coprocessor pre-close hooks at 1732014438066Disabling compacts and flushes for region at 1732014438066Disabling writes for close at 1732014438066Obtaining lock to block concurrent updates at 1732014438066Preparing flush snapshotting stores in 295075fcca1bb094ad7be004ade119ff at 1732014438066Finished memstore snapshotting test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732014438066Flushing stores of test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. at 1732014438067 (+1 ms)Flushing 295075fcca1bb094ad7be004ade119ff/f2: creating writer at 1732014438067Flushing 295075fcca1bb094ad7be004ade119ff/f2: appending metadata at 1732014438079 (+12 ms)Flushing 295075fcca1bb094ad7be004ade119ff/f2: closing flushed file at 1732014438079Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39fa214d: reopening flushed file at 1732014438489 (+410 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 295075fcca1bb094ad7be004ade119ff in 430ms, sequenceid=348, compaction requested=false at 1732014438496 (+7 ms)Writing region close event to WAL at 1732014438497 (+1 ms)Running coprocessor post-close hooks at 1732014438501 (+4 ms)Closed at 1732014438501 2024-11-19T11:07:18,501 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732014341354,,1732014341848.295075fcca1bb094ad7be004ade119ff. 2024-11-19T11:07:18,502 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing af7e2bb446c6a1468013d39e5733df3e, disabling compactions & flushes 2024-11-19T11:07:18,502 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:07:18,502 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:07:18,502 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. after waiting 0 ms 2024-11-19T11:07:18,502 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:07:18,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741980_1156 (size=5866) 2024-11-19T11:07:18,505 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/ns3/test-17320143413542/af7e2bb446c6a1468013d39e5733df3e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:18,506 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for af7e2bb446c6a1468013d39e5733df3e: Waiting for close lock at 1732014438502Running coprocessor pre-close hooks at 1732014438502Disabling compacts and flushes for region at 1732014438502Disabling writes for close at 1732014438502Writing region close event to WAL at 1732014438502Running coprocessor post-close hooks at 1732014438506 (+4 ms)Closed at 1732014438506 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17320143413542,,1732014346504.af7e2bb446c6a1468013d39e5733df3e. 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e1b8b050c5202951d3c6975654ccd6c4, disabling compactions & flushes 2024-11-19T11:07:18,506 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. after waiting 0 ms 2024-11-19T11:07:18,506 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:07:18,509 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/backup/system_bulk/e1b8b050c5202951d3c6975654ccd6c4/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-19T11:07:18,510 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:18,510 INFO [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:07:18,510 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e1b8b050c5202951d3c6975654ccd6c4: Waiting for close lock at 1732014438506Running coprocessor pre-close hooks at 1732014438506Disabling compacts and flushes for region at 1732014438506Disabling writes for close at 1732014438506Writing region close event to WAL at 1732014438507 (+1 ms)Running coprocessor post-close hooks at 1732014438510 (+3 ms)Closed at 1732014438510 2024-11-19T11:07:18,510 DEBUG [RS_CLOSE_REGION-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732014354419.e1b8b050c5202951d3c6975654ccd6c4. 2024-11-19T11:07:18,618 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T11:07:18,618 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T11:07:18,618 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:07:18,818 DEBUG [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T11:07:18,906 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/table/ceaadff730324d4e92d1d84705231869 2024-11-19T11:07:18,912 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/info/6d8b333fcf85414dbd2ca1bd30a7f6dd as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/info/6d8b333fcf85414dbd2ca1bd30a7f6dd 2024-11-19T11:07:18,917 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/info/6d8b333fcf85414dbd2ca1bd30a7f6dd, entries=69, sequenceid=59, filesize=13.8 K 2024-11-19T11:07:18,918 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/ns/e79267a41d41471d9ed8927c2563f719 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/ns/e79267a41d41471d9ed8927c2563f719 2024-11-19T11:07:18,924 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/ns/e79267a41d41471d9ed8927c2563f719, entries=7, sequenceid=59, filesize=5.2 K 2024-11-19T11:07:18,925 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/.tmp/table/ceaadff730324d4e92d1d84705231869 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/table/ceaadff730324d4e92d1d84705231869 2024-11-19T11:07:18,930 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/table/ceaadff730324d4e92d1d84705231869, entries=13, sequenceid=59, filesize=5.7 K 2024-11-19T11:07:18,931 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1315ms, sequenceid=59, compaction requested=false 2024-11-19T11:07:18,936 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-19T11:07:18,936 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:18,936 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T11:07:18,936 INFO [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T11:07:18,936 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732014437616Running coprocessor pre-close hooks at 1732014437616Disabling compacts and flushes for region at 1732014437616Disabling writes for close at 1732014437616Obtaining lock to block concurrent updates at 1732014437616Preparing flush snapshotting stores in 1588230740 at 1732014437616Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1732014437617 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732014437617Flushing 1588230740/info: creating writer at 1732014437617Flushing 1588230740/info: appending metadata at 1732014437637 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732014437637Flushing 1588230740/ns: creating writer at 1732014438047 (+410 ms)Flushing 1588230740/ns: appending metadata at 1732014438062 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732014438063 (+1 ms)Flushing 1588230740/table: creating writer at 1732014438478 (+415 ms)Flushing 1588230740/table: appending metadata at 1732014438501 (+23 ms)Flushing 1588230740/table: closing flushed file at 1732014438501Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@686d226d: reopening flushed file at 1732014438911 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2df8f13e: reopening flushed file at 1732014438917 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@112f298c: reopening flushed file at 1732014438924 (+7 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1315ms, sequenceid=59, compaction requested=false at 1732014438931 (+7 ms)Writing region close event to WAL at 1732014438933 (+2 ms)Running coprocessor post-close hooks at 1732014438936 (+3 ms)Closed at 1732014438936 2024-11-19T11:07:18,937 DEBUG [RS_CLOSE_META-regionserver/c28903dc2d16:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T11:07:19,018 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(976): stopping server c28903dc2d16,43005,1732014332227; all regions closed. 2024-11-19T11:07:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741954_1130 (size=3260) 2024-11-19T11:07:19,022 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.meta.1732014426676.meta not finished, retry = 0 2024-11-19T11:07:19,136 DEBUG [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs 2024-11-19T11:07:19,136 INFO [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.meta:.meta(num 1732014426676) 2024-11-19T11:07:19,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741956_1132 (size=5275) 2024-11-19T11:07:19,143 DEBUG [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs 2024-11-19T11:07:19,143 INFO [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-1:(num 1732014426715) 2024-11-19T11:07:19,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741955_1131 (size=1037) 2024-11-19T11:07:19,144 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/WALs/c28903dc2d16,43005,1732014332227/c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0.1732014426699 not finished, retry = 0 2024-11-19T11:07:19,248 DEBUG [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/oldWALs 2024-11-19T11:07:19,249 INFO [RS:0;c28903dc2d16:43005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c28903dc2d16%2C43005%2C1732014332227.c28903dc2d16%2C43005%2C1732014332227.regiongroup-0:(num 1732014426699) 2024-11-19T11:07:19,249 DEBUG [RS:0;c28903dc2d16:43005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:19,249 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T11:07:19,249 INFO [RS:0;c28903dc2d16:43005 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T11:07:19,249 INFO [RS:0;c28903dc2d16:43005 {}] hbase.ChoreService(370): Chore service for: regionserver/c28903dc2d16:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T11:07:19,249 INFO [RS:0;c28903dc2d16:43005 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T11:07:19,250 INFO [regionserver/c28903dc2d16:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T11:07:19,250 INFO [RS:0;c28903dc2d16:43005 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43005 2024-11-19T11:07:19,336 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-19T11:07:19,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-19T11:07:19,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/c28903dc2d16,43005,1732014332227 2024-11-19T11:07:19,410 INFO [RS:0;c28903dc2d16:43005 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T11:07:19,412 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c28903dc2d16,43005,1732014332227] 2024-11-19T11:07:19,434 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/c28903dc2d16,43005,1732014332227 already deleted, retry=false 2024-11-19T11:07:19,435 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c28903dc2d16,43005,1732014332227 expired; onlineServers=0 2024-11-19T11:07:19,435 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c28903dc2d16,39427,1732014330695' ***** 2024-11-19T11:07:19,435 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T11:07:19,435 INFO [M:0;c28903dc2d16:39427 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-19T11:07:19,440 INFO [M:0;c28903dc2d16:39427 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@1190ad88{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-19T11:07:19,441 INFO [M:0;c28903dc2d16:39427 {}] server.AbstractConnector(383): Stopped ServerConnector@28c6e73f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-19T11:07:19,441 INFO [M:0;c28903dc2d16:39427 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:19,441 INFO [M:0;c28903dc2d16:39427 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5f79216d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-19T11:07:19,441 INFO [M:0;c28903dc2d16:39427 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2f4c8752{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:19,442 INFO [M:0;c28903dc2d16:39427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T11:07:19,442 INFO [M:0;c28903dc2d16:39427 {}] master.HMaster(1770): Stopping master jetty server 2024-11-19T11:07:19,442 INFO [M:0;c28903dc2d16:39427 {}] server.AbstractConnector(383): Stopped ServerConnector@39c3cce8{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-19T11:07:19,443 INFO [M:0;c28903dc2d16:39427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T11:07:19,443 DEBUG [M:0;c28903dc2d16:39427 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T11:07:19,443 INFO [M:0;c28903dc2d16:39427 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-19T11:07:19,443 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T11:07:19,443 INFO [M:0;c28903dc2d16:39427 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;c28903dc2d16:39427. 2024-11-19T11:07:19,443 DEBUG [master/c28903dc2d16:0:becomeActiveMaster-HFileCleaner.large.0-1732014335328 {}] cleaner.HFileCleaner(306): Exit Thread[master/c28903dc2d16:0:becomeActiveMaster-HFileCleaner.large.0-1732014335328,5,FailOnTimeoutGroup] 2024-11-19T11:07:19,443 DEBUG [master/c28903dc2d16:0:becomeActiveMaster-HFileCleaner.small.0-1732014335329 {}] cleaner.HFileCleaner(306): Exit Thread[master/c28903dc2d16:0:becomeActiveMaster-HFileCleaner.small.0-1732014335329,5,FailOnTimeoutGroup] 2024-11-19T11:07:19,443 DEBUG [M:0;c28903dc2d16:39427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T11:07:19,443 DEBUG [M:0;c28903dc2d16:39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:19,443 DEBUG [M:0;c28903dc2d16:39427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T11:07:19,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T11:07:19,443 DEBUG [M:0;c28903dc2d16:39427 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T11:07:19,443 INFO [M:0;c28903dc2d16:39427 {}] hbase.ChoreService(370): Chore service for: master/c28903dc2d16:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T11:07:19,444 INFO [M:0;c28903dc2d16:39427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T11:07:19,444 DEBUG [M:0;c28903dc2d16:39427 {}] master.HMaster(1795): Stopping service threads 2024-11-19T11:07:19,444 INFO [M:0;c28903dc2d16:39427 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T11:07:19,444 INFO [M:0;c28903dc2d16:39427 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T11:07:19,444 INFO [M:0;c28903dc2d16:39427 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T11:07:19,445 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T11:07:19,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-19T11:07:19,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-19T11:07:19,445 DEBUG [M:0;c28903dc2d16:39427 {}] zookeeper.ZKUtil(347): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-19T11:07:19,445 WARN [M:0;c28903dc2d16:39427 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T11:07:19,446 INFO [M:0;c28903dc2d16:39427 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/.lastflushedseqids 2024-11-19T11:07:19,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741981_1157 (size=495) 2024-11-19T11:07:19,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:07:19,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43005-0x10152c54b450001, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:07:19,525 INFO [RS:0;c28903dc2d16:43005 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T11:07:19,525 INFO [RS:0;c28903dc2d16:43005 {}] regionserver.HRegionServer(1031): Exiting; stopping=c28903dc2d16,43005,1732014332227; zookeeper connection closed. 2024-11-19T11:07:19,526 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58782b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58782b 2024-11-19T11:07:19,526 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T11:07:19,857 INFO [M:0;c28903dc2d16:39427 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T11:07:19,858 INFO [M:0;c28903dc2d16:39427 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T11:07:19,858 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T11:07:19,876 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:07:19,876 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:07:19,876 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T11:07:19,876 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:07:19,876 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=319.21 KB heapSize=375.29 KB 2024-11-19T11:07:19,890 DEBUG [M:0;c28903dc2d16:39427 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8e31aaf0a98141ccaf447d8abd2ebe81 is 82, key is hbase:meta,,1/info:regioninfo/1732014336663/Put/seqid=0 2024-11-19T11:07:19,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741982_1158 (size=5672) 2024-11-19T11:07:20,296 INFO [M:0;c28903dc2d16:39427 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8e31aaf0a98141ccaf447d8abd2ebe81 2024-11-19T11:07:20,326 DEBUG [M:0;c28903dc2d16:39427 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97ac54978fab446495224fbb5779848b is 2184, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1732014386602/Put/seqid=0 2024-11-19T11:07:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741983_1159 (size=26516) 2024-11-19T11:07:20,732 INFO [M:0;c28903dc2d16:39427 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.66 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97ac54978fab446495224fbb5779848b 2024-11-19T11:07:20,743 INFO [M:0;c28903dc2d16:39427 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 97ac54978fab446495224fbb5779848b 2024-11-19T11:07:20,760 DEBUG [M:0;c28903dc2d16:39427 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b6082b56cd83446186db5d1c03eccc2f is 69, key is c28903dc2d16,43005,1732014332227/rs:state/1732014335446/Put/seqid=0 2024-11-19T11:07:20,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741984_1160 (size=5156) 2024-11-19T11:07:21,059 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T11:07:21,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-19T11:07:21,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T11:07:21,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-19T11:07:21,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17320143413542 2024-11-19T11:07:21,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17320143413543 2024-11-19T11:07:21,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732014341354 2024-11-19T11:07:21,061 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17320143413541 2024-11-19T11:07:21,061 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-19T11:07:21,167 INFO [M:0;c28903dc2d16:39427 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b6082b56cd83446186db5d1c03eccc2f 2024-11-19T11:07:21,177 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8e31aaf0a98141ccaf447d8abd2ebe81 as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8e31aaf0a98141ccaf447d8abd2ebe81 2024-11-19T11:07:21,184 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8e31aaf0a98141ccaf447d8abd2ebe81, entries=8, sequenceid=667, filesize=5.5 K 2024-11-19T11:07:21,185 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/97ac54978fab446495224fbb5779848b as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/97ac54978fab446495224fbb5779848b 2024-11-19T11:07:21,190 INFO [M:0;c28903dc2d16:39427 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 97ac54978fab446495224fbb5779848b 2024-11-19T11:07:21,191 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/97ac54978fab446495224fbb5779848b, entries=69, sequenceid=667, filesize=25.9 K 2024-11-19T11:07:21,192 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b6082b56cd83446186db5d1c03eccc2f as hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b6082b56cd83446186db5d1c03eccc2f 2024-11-19T11:07:21,198 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35831/user/jenkins/test-data/e78a24bd-80e9-8625-d96d-2150a355e442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b6082b56cd83446186db5d1c03eccc2f, entries=1, sequenceid=667, filesize=5.0 K 2024-11-19T11:07:21,199 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(3140): Finished flush of dataSize ~319.21 KB/326876, heapSize ~374.99 KB/383992, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1323ms, sequenceid=667, compaction requested=false 2024-11-19T11:07:21,200 INFO [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T11:07:21,200 DEBUG [M:0;c28903dc2d16:39427 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732014439858Disabling compacts and flushes for region at 1732014439858Disabling writes for close at 1732014439876 (+18 ms)Obtaining lock to block concurrent updates at 1732014439876Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732014439876Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326876, getHeapSize=384232, getOffHeapSize=0, getCellsCount=746 at 1732014439876Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732014439877 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732014439877Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732014439890 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732014439890Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732014440308 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732014440325 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732014440325Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732014440744 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732014440760 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732014440760Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63c08e21: reopening flushed file at 1732014441176 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69e51e3e: reopening flushed file at 1732014441184 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4819ff37: reopening flushed file at 1732014441191 (+7 ms)Finished flush of dataSize ~319.21 KB/326876, heapSize ~374.99 KB/383992, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1323ms, sequenceid=667, compaction requested=false at 1732014441199 (+8 ms)Writing region close event to WAL at 1732014441200 (+1 ms)Closed at 1732014441200 2024-11-19T11:07:21,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37071 is added to blk_1073741830_1006 (size=370966) 2024-11-19T11:07:21,204 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T11:07:21,204 INFO [M:0;c28903dc2d16:39427 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T11:07:21,204 INFO [M:0;c28903dc2d16:39427 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-19T11:07:21,205 INFO [M:0;c28903dc2d16:39427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39427 2024-11-19T11:07:21,205 INFO [M:0;c28903dc2d16:39427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T11:07:21,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:07:21,321 INFO [M:0;c28903dc2d16:39427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T11:07:21,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39427-0x10152c54b450000, quorum=127.0.0.1:51477, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T11:07:21,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@161bd8f9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T11:07:21,337 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14a6ba7a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:07:21,337 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:21,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27593828{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T11:07:21,338 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fdab1fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:21,340 WARN [BP-264477037-172.17.0.2-1732014326384 heartbeating to localhost/127.0.0.1:35831 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T11:07:21,340 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T11:07:21,340 WARN [BP-264477037-172.17.0.2-1732014326384 heartbeating to localhost/127.0.0.1:35831 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-264477037-172.17.0.2-1732014326384 (Datanode Uuid 2afb7002-8414-4297-b9f4-c050734cf990) service to localhost/127.0.0.1:35831 2024-11-19T11:07:21,340 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T11:07:21,341 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/data/data1/current/BP-264477037-172.17.0.2-1732014326384 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T11:07:21,342 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/cluster_72b20095-94cc-f75b-a256-60bff8fc9566/data/data2/current/BP-264477037-172.17.0.2-1732014326384 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T11:07:21,342 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T11:07:21,348 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4084a5e6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T11:07:21,348 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3151d644{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:07:21,349 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:21,349 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4edc5bd1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T11:07:21,349 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c623081{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:21,358 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T11:07:21,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T11:07:21,407 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-19T11:07:21,411 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7742a1dc{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-19T11:07:21,412 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@42cd877f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:07:21,412 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:21,412 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@942d7cc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-19T11:07:21,412 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63b686e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:21,420 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d2f5bb7{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-19T11:07:21,420 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b9922e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:07:21,420 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:21,420 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b9502e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-19T11:07:21,420 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9e3e547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:21,424 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-19T11:07:21,424 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17ec89fc{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-19T11:07:21,425 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@403cdf84{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:07:21,425 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:21,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f41edce{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-19T11:07:21,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a1040a0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:21,427 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-19T11:07:21,430 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-19T11:07:21,430 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-19T11:07:21,431 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-19T11:07:21,433 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f51081{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-19T11:07:21,434 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c920e7f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T11:07:21,434 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T11:07:21,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d8c4596{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-19T11:07:21,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68f08892{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/2ef83cec-757d-deeb-3498-b20bc6c4cabe/hadoop.log.dir/,STOPPED} 2024-11-19T11:07:21,436 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-19T11:07:21,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped