2024-11-12 15:12:11,840 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-12 15:12:11,860 main DEBUG Took 0.017093 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 15:12:11,861 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 15:12:11,861 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 15:12:11,863 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 15:12:11,865 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,882 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 15:12:11,902 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,906 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,907 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,907 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,909 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,910 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,912 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,914 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,916 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,917 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,917 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,918 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,920 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,921 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 15:12:11,922 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,923 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 15:12:11,925 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 15:12:11,927 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 15:12:11,930 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 15:12:11,931 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 15:12:11,933 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 15:12:11,934 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 15:12:11,948 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 15:12:11,952 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 15:12:11,955 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 15:12:11,956 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 15:12:11,956 main DEBUG createAppenders(={Console}) 2024-11-12 15:12:11,958 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-12 15:12:11,958 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-12 15:12:11,959 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-12 15:12:11,959 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 15:12:11,960 main DEBUG OutputStream closed 2024-11-12 15:12:11,960 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 15:12:11,961 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 15:12:11,961 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-12 15:12:12,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 15:12:12,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 15:12:12,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 15:12:12,084 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 15:12:12,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 15:12:12,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 15:12:12,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 15:12:12,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 15:12:12,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 15:12:12,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 15:12:12,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 15:12:12,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 15:12:12,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 15:12:12,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 15:12:12,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 15:12:12,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 15:12:12,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 15:12:12,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 15:12:12,099 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 15:12:12,099 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-12 15:12:12,100 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 15:12:12,102 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-12T15:12:12,161 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-12 15:12:12,166 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 15:12:12,166 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T15:12:12,501 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6 2024-11-12T15:12:12,505 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-12T15:12:12,507 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:12:12,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T15:12:12,522 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01, deleteOnExit=true 2024-11-12T15:12:12,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T15:12:12,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/test.cache.data in system properties and HBase conf 2024-11-12T15:12:12,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T15:12:12,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir in system properties and HBase conf 2024-11-12T15:12:12,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T15:12:12,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T15:12:12,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T15:12:12,620 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T15:12:12,755 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T15:12:12,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T15:12:12,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T15:12:12,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T15:12:12,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T15:12:12,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T15:12:12,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T15:12:12,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T15:12:12,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T15:12:12,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T15:12:12,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/nfs.dump.dir in system properties and HBase conf 2024-11-12T15:12:12,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir in system properties and HBase conf 2024-11-12T15:12:12,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T15:12:12,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T15:12:12,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T15:12:14,107 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T15:12:14,248 INFO [Time-limited test {}] log.Log(170): Logging initialized @3427ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T15:12:14,382 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:14,494 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T15:12:14,537 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:14,538 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:14,540 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T15:12:14,560 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:14,564 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3cb0c733{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:14,566 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@282f8860{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T15:12:14,803 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c784fd0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir/jetty-localhost-41267-hadoop-hdfs-3_4_1-tests_jar-_-any-18009324447483407403/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T15:12:14,814 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2fdb04ed{HTTP/1.1, (http/1.1)}{localhost:41267} 2024-11-12T15:12:14,814 INFO [Time-limited test {}] server.Server(415): Started @3994ms 2024-11-12T15:12:16,397 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:16,408 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T15:12:16,412 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:16,413 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:16,413 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T15:12:16,419 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:16,420 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T15:12:16,571 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@cb44ad4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir/jetty-localhost-34805-hadoop-hdfs-3_4_1-tests_jar-_-any-15919237690803269300/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T15:12:16,572 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:34805} 2024-11-12T15:12:16,573 INFO [Time-limited test {}] server.Server(415): Started @5752ms 2024-11-12T15:12:16,681 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T15:12:18,061 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/data/data2/current/BP-1953189042-172.17.0.3-1731424333465/current, will proceed with Du for space computation calculation, 2024-11-12T15:12:18,061 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/data/data1/current/BP-1953189042-172.17.0.3-1731424333465/current, will proceed with Du for space computation calculation, 2024-11-12T15:12:18,109 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T15:12:18,157 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84032c389dd7b545 with lease ID 0x53212984f8a9763c: Processing first storage report for DS-e886ce3b-9173-4f04-a761-735839061db7 from datanode DatanodeRegistration(127.0.0.1:33325, datanodeUuid=6317909c-bf35-4176-afcd-944ba4e67199, infoPort=37305, infoSecurePort=0, ipcPort=36827, storageInfo=lv=-57;cid=testClusterID;nsid=392298151;c=1731424333465) 2024-11-12T15:12:18,158 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84032c389dd7b545 with lease ID 0x53212984f8a9763c: from storage DS-e886ce3b-9173-4f04-a761-735839061db7 node DatanodeRegistration(127.0.0.1:33325, datanodeUuid=6317909c-bf35-4176-afcd-944ba4e67199, infoPort=37305, infoSecurePort=0, ipcPort=36827, storageInfo=lv=-57;cid=testClusterID;nsid=392298151;c=1731424333465), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T15:12:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84032c389dd7b545 with lease ID 0x53212984f8a9763c: Processing first storage report for DS-ea2193b8-f956-479d-b763-6b9aac22fec9 from datanode DatanodeRegistration(127.0.0.1:33325, datanodeUuid=6317909c-bf35-4176-afcd-944ba4e67199, infoPort=37305, infoSecurePort=0, ipcPort=36827, storageInfo=lv=-57;cid=testClusterID;nsid=392298151;c=1731424333465) 2024-11-12T15:12:18,159 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84032c389dd7b545 with lease ID 0x53212984f8a9763c: from storage DS-ea2193b8-f956-479d-b763-6b9aac22fec9 node DatanodeRegistration(127.0.0.1:33325, datanodeUuid=6317909c-bf35-4176-afcd-944ba4e67199, infoPort=37305, infoSecurePort=0, ipcPort=36827, storageInfo=lv=-57;cid=testClusterID;nsid=392298151;c=1731424333465), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T15:12:18,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6 2024-11-12T15:12:18,379 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/zookeeper_0, clientPort=51996, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T15:12:18,390 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51996 2024-11-12T15:12:18,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:18,409 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:18,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741825_1001 (size=7) 2024-11-12T15:12:19,107 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 with version=8 2024-11-12T15:12:19,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/hbase-staging 2024-11-12T15:12:19,121 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-12T15:12:19,121 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-12T15:12:19,121 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-12T15:12:19,121 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-12T15:12:19,203 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T15:12:19,523 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2b6d221c5cde:0 server-side Connection retries=45 2024-11-12T15:12:19,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:12:19,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T15:12:19,540 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T15:12:19,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:12:19,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T15:12:19,720 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T15:12:19,796 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T15:12:19,809 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T15:12:19,814 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T15:12:19,849 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 143076 (auto-detected) 2024-11-12T15:12:19,851 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-12T15:12:19,879 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43085 2024-11-12T15:12:19,910 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43085 connecting to ZooKeeper ensemble=127.0.0.1:51996 2024-11-12T15:12:20,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430850x0, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T15:12:20,075 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43085-0x1012f9abd630000 connected 2024-11-12T15:12:20,172 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:20,174 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:20,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T15:12:20,210 INFO [Time-limited test {}] log.Log(170): Logging initialized @9390ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T15:12:20,256 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-12T15:12:20,256 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-12T15:12:20,257 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-12T15:12:20,257 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-12T15:12:20,258 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-12T15:12:20,260 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-12T15:12:20,260 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-12T15:12:20,260 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-12T15:12:20,262 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-12T15:12:20,344 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 39549 2024-11-12T15:12:20,346 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-12T15:12:20,369 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T15:12:20,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4cdc734c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:20,372 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T15:12:20,373 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@e95be18{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-12T15:12:20,433 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-12T15:12:20,444 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:20,444 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:20,445 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T15:12:20,451 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T15:12:21,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2aafda76{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-12T15:12:21,204 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20f7d02c{HTTP/1.1, (http/1.1)}{0.0.0.0:39549} 2024-11-12T15:12:21,204 INFO [Time-limited test {}] server.Server(415): Started @10384ms 2024-11-12T15:12:21,210 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265, hbase.cluster.distributed=false 2024-11-12T15:12:21,242 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-12T15:12:21,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43085 2024-11-12T15:12:21,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43085 2024-11-12T15:12:21,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43085 2024-11-12T15:12:21,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43085 2024-11-12T15:12:21,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43085 2024-11-12T15:12:21,365 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2b6d221c5cde:0 server-side Connection retries=45 2024-11-12T15:12:21,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:12:21,368 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T15:12:21,368 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T15:12:21,368 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T15:12:21,368 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T15:12:21,372 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T15:12:21,376 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T15:12:21,377 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34071 2024-11-12T15:12:21,379 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34071 connecting to ZooKeeper ensemble=127.0.0.1:51996 2024-11-12T15:12:21,380 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:21,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:21,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340710x0, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T15:12:21,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34071-0x1012f9abd630001 connected 2024-11-12T15:12:21,487 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T15:12:21,490 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-12T15:12:21,490 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-12T15:12:21,491 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-12T15:12:21,491 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-12T15:12:21,492 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-12T15:12:21,493 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-12T15:12:21,493 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-12T15:12:21,494 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-12T15:12:21,494 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-12T15:12:21,497 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 42503 2024-11-12T15:12:21,497 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-12T15:12:21,499 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T15:12:21,499 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@21ae8981{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:21,500 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T15:12:21,500 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@126a703e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-12T15:12:21,520 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-12T15:12:21,522 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:21,522 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:21,522 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T15:12:21,524 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-12T15:12:21,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@7253afc7{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-12T15:12:21,529 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b99b3c7{HTTP/1.1, (http/1.1)}{0.0.0.0:42503} 2024-11-12T15:12:21,529 INFO [Time-limited test {}] server.Server(415): Started @10709ms 2024-11-12T15:12:21,535 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T15:12:21,545 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T15:12:21,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-12T15:12:21,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-12T15:12:21,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34071 2024-11-12T15:12:21,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34071 2024-11-12T15:12:21,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34071 2024-11-12T15:12:21,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34071 2024-11-12T15:12:21,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34071 2024-11-12T15:12:21,579 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2b6d221c5cde:43085 2024-11-12T15:12:21,583 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-12T15:12:21,589 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@629f2090{HTTP/1.1, (http/1.1)}{0.0.0.0:43935} 2024-11-12T15:12:21,590 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] server.Server(415): Started @10769ms 2024-11-12T15:12:21,590 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:21,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T15:12:21,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T15:12:21,604 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:21,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-12T15:12:21,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:21,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:21,634 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-12T15:12:21,636 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/2b6d221c5cde,43085,1731424339322 from backup master directory 2024-11-12T15:12:21,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T15:12:21,645 WARN [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T15:12:21,645 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:21,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:21,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-12T15:12:21,647 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T15:12:21,649 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T15:12:21,702 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/hbase.id] with ID: 1c1f44d9-3bb0-42bc-892d-04aff690dcc3 2024-11-12T15:12:21,703 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.tmp/hbase.id 2024-11-12T15:12:21,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741826_1002 (size=42) 2024-11-12T15:12:22,115 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.tmp/hbase.id]:[hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/hbase.id] 2024-11-12T15:12:22,180 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T15:12:22,187 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T15:12:22,215 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-12T15:12:22,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:22,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:22,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741827_1003 (size=196) 2024-11-12T15:12:22,717 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:22,720 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T15:12:22,728 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-12T15:12:22,732 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-12T15:12:22,747 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T15:12:22,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741828_1004 (size=1189) 2024-11-12T15:12:23,212 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store 2024-11-12T15:12:23,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741829_1005 (size=34) 2024-11-12T15:12:23,641 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T15:12:23,645 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:23,647 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T15:12:23,647 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:12:23,647 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:12:23,649 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T15:12:23,649 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:12:23,650 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:12:23,651 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731424343646Disabling compacts and flushes for region at 1731424343646Disabling writes for close at 1731424343649 (+3 ms)Writing region close event to WAL at 1731424343650 (+1 ms)Closed at 1731424343650 2024-11-12T15:12:23,654 WARN [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/.initializing 2024-11-12T15:12:23,654 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/WALs/2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:23,660 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:12:23,668 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 2b6d221c5cde,43085,1731424339322.regiongroup-0 2024-11-12T15:12:23,689 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C43085%2C1731424339322.2b6d221c5cde%2C43085%2C1731424339322.regiongroup-0, suffix=, logDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/WALs/2b6d221c5cde,43085,1731424339322, archiveDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/oldWALs, maxLogs=10 2024-11-12T15:12:23,722 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/WALs/2b6d221c5cde,43085,1731424339322/2b6d221c5cde%2C43085%2C1731424339322.2b6d221c5cde%2C43085%2C1731424339322.regiongroup-0.1731424343695, exclude list is [], retry=0 2024-11-12T15:12:23,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:23,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T15:12:23,785 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/WALs/2b6d221c5cde,43085,1731424339322/2b6d221c5cde%2C43085%2C1731424339322.2b6d221c5cde%2C43085%2C1731424339322.regiongroup-0.1731424343695 2024-11-12T15:12:23,787 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:23,787 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:23,788 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:23,792 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,794 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T15:12:23,866 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:23,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:23,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,874 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T15:12:23,874 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:23,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:23,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T15:12:23,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:23,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:23,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T15:12:23,886 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:23,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:23,888 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,892 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,893 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,900 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,901 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,905 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T15:12:23,910 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T15:12:23,916 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:23,918 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10434607040, jitterRate=-0.02820149064064026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T15:12:23,928 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731424343811Initializing all the Stores at 1731424343813 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424343813Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424343815 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424343815Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424343815Cleaning up temporary data from old regions at 1731424343901 (+86 ms)Region opened successfully at 1731424343928 (+27 ms) 2024-11-12T15:12:23,930 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T15:12:23,965 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e73476f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=2b6d221c5cde/172.17.0.3:0 2024-11-12T15:12:24,005 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T15:12:24,018 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T15:12:24,018 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T15:12:24,021 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T15:12:24,023 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T15:12:24,030 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-12T15:12:24,031 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T15:12:24,068 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T15:12:24,084 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-12T15:12:24,333 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-12T15:12:24,337 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T15:12:24,339 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-12T15:12:24,409 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-12T15:12:24,412 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T15:12:24,416 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-12T15:12:24,422 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-12T15:12:24,424 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-12T15:12:24,433 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-12T15:12:24,452 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T15:12:24,464 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-12T15:12:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-12T15:12:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-12T15:12:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,478 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2b6d221c5cde,43085,1731424339322, sessionid=0x1012f9abd630000, setting cluster-up flag (Was=false) 2024-11-12T15:12:24,485 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-12T15:12:24,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,538 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-12T15:12:24,541 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:24,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,693 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-12T15:12:24,695 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:24,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:24,749 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-12T15:12:24,751 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:24,759 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T15:12:24,771 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(746): ClusterId : 1c1f44d9-3bb0-42bc-892d-04aff690dcc3 2024-11-12T15:12:24,772 INFO [RS:0;2b6d221c5cde:34071 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-12T15:12:24,796 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T15:12:24,819 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T15:12:24,819 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-12T15:12:24,836 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-12T15:12:24,836 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T15:12:24,845 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T15:12:24,846 DEBUG [RS:0;2b6d221c5cde:34071 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3ebb3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=2b6d221c5cde/172.17.0.3:0 2024-11-12T15:12:24,865 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2b6d221c5cde:34071 2024-11-12T15:12:24,869 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T15:12:24,869 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T15:12:24,869 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T15:12:24,871 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(2659): reportForDuty to master=2b6d221c5cde,43085,1731424339322 with port=34071, startcode=1731424341329 2024-11-12T15:12:24,885 DEBUG [RS:0;2b6d221c5cde:34071 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T15:12:24,895 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T15:12:24,907 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T15:12:24,915 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T15:12:24,921 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2b6d221c5cde,43085,1731424339322 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T15:12:24,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37481, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T15:12:24,931 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43085 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T15:12:24,932 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:12:24,932 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:12:24,932 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:12:24,932 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T15:12:24,932 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2b6d221c5cde:0, corePoolSize=10, maxPoolSize=10 2024-11-12T15:12:24,933 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:24,933 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=2, maxPoolSize=2 2024-11-12T15:12:24,933 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:24,959 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T15:12:24,960 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T15:12:24,964 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T15:12:24,964 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731424374964 2024-11-12T15:12:24,965 WARN [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T15:12:24,966 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T15:12:24,966 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:24,967 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T15:12:24,968 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T15:12:24,974 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T15:12:24,974 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-12T15:12:24,975 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T15:12:24,975 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T15:12:24,976 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T15:12:24,976 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:24,981 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T15:12:24,982 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T15:12:24,983 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T15:12:24,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741831_1007 (size=1321) 2024-11-12T15:12:24,992 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T15:12:24,993 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:24,999 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ca1ae46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:24,999 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:25,003 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:25,010 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:25,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741832_1008 (size=32) 2024-11-12T15:12:25,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:25,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:25,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T15:12:25,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:25,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b0b5cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:25,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:25,020 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T15:12:25,020 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:25,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T15:12:25,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:25,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:25,026 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T15:12:25,026 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T15:12:25,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c90f8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:25,030 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-12T15:12:25,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T15:12:25,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T15:12:25,036 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T15:12:25,036 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T15:12:25,036 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,037 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T15:12:25,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T15:12:25,040 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.large.0-1731424345038,5,FailOnTimeoutGroup] 2024-11-12T15:12:25,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740 2024-11-12T15:12:25,041 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.small.0-1731424345040,5,FailOnTimeoutGroup] 2024-11-12T15:12:25,041 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,041 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T15:12:25,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740 2024-11-12T15:12:25,043 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,043 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T15:12:25,045 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T15:12:25,046 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T15:12:25,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T15:12:25,055 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:25,057 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12077160160, jitterRate=0.12477318942546844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T15:12:25,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731424345013Initializing all the Stores at 1731424345016 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424345016Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424345016Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424345016Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424345016Cleaning up temporary data from old regions at 1731424345045 (+29 ms)Region opened successfully at 1731424345061 (+16 ms) 2024-11-12T15:12:25,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T15:12:25,062 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T15:12:25,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T15:12:25,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T15:12:25,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T15:12:25,063 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T15:12:25,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731424345062Disabling compacts and flushes for region at 1731424345062Disabling writes for close at 1731424345062Writing region close event to WAL at 1731424345063 (+1 ms)Closed at 1731424345063 2024-11-12T15:12:25,066 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(2659): reportForDuty to master=2b6d221c5cde,43085,1731424339322 with port=34071, startcode=1731424341329 2024-11-12T15:12:25,069 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43085 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,071 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43085 {}] master.ServerManager(517): Registering regionserver=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,074 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T15:12:25,074 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T15:12:25,081 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:25,081 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33269 2024-11-12T15:12:25,081 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=39549 2024-11-12T15:12:25,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T15:12:25,093 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T15:12:25,097 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T15:12:25,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-12T15:12:25,149 DEBUG [RS:0;2b6d221c5cde:34071 {}] zookeeper.ZKUtil(111): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rs/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,150 WARN [RS:0;2b6d221c5cde:34071 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T15:12:25,150 INFO [RS:0;2b6d221c5cde:34071 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-12T15:12:25,150 INFO [RS:0;2b6d221c5cde:34071 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-12T15:12:25,150 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,153 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2b6d221c5cde,34071,1731424341329] 2024-11-12T15:12:25,180 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T15:12:25,201 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T15:12:25,208 INFO [RS:0;2b6d221c5cde:34071 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T15:12:25,208 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,209 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T15:12:25,216 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T15:12:25,218 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,219 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,219 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,219 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,219 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,219 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,219 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2b6d221c5cde:0, corePoolSize=2, maxPoolSize=2 2024-11-12T15:12:25,220 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,220 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,220 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,220 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,220 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,220 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T15:12:25,221 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0, corePoolSize=3, maxPoolSize=3 2024-11-12T15:12:25,221 DEBUG [RS:0;2b6d221c5cde:34071 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0, corePoolSize=3, maxPoolSize=3 2024-11-12T15:12:25,222 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,222 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,222 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,222 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,223 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,223 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,34071,1731424341329-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T15:12:25,248 WARN [2b6d221c5cde:43085 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-12T15:12:25,256 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T15:12:25,259 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,34071,1731424341329-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,260 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,260 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.Replication(171): 2b6d221c5cde,34071,1731424341329 started 2024-11-12T15:12:25,286 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:25,287 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1482): Serving as 2b6d221c5cde,34071,1731424341329, RpcServer on 2b6d221c5cde/172.17.0.3:34071, sessionid=0x1012f9abd630001 2024-11-12T15:12:25,288 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T15:12:25,288 DEBUG [RS:0;2b6d221c5cde:34071 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,288 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2b6d221c5cde,34071,1731424341329' 2024-11-12T15:12:25,289 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-12T15:12:25,290 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-12T15:12:25,290 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T15:12:25,291 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-12T15:12:25,291 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2b6d221c5cde,34071,1731424341329' 2024-11-12T15:12:25,291 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:12:25,291 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:12:25,292 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-12T15:12:25,292 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-12T15:12:25,292 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T15:12:25,292 DEBUG [RS:0;2b6d221c5cde:34071 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,292 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2b6d221c5cde,34071,1731424341329' 2024-11-12T15:12:25,292 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-12T15:12:25,293 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-12T15:12:25,294 DEBUG [RS:0;2b6d221c5cde:34071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T15:12:25,294 INFO [RS:0;2b6d221c5cde:34071 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T15:12:25,294 INFO [RS:0;2b6d221c5cde:34071 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T15:12:25,400 INFO [RS:0;2b6d221c5cde:34071 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:12:25,401 INFO [RS:0;2b6d221c5cde:34071 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 2b6d221c5cde,34071,1731424341329.regiongroup-0 2024-11-12T15:12:25,405 INFO [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0, suffix=, logDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329, archiveDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs, maxLogs=32 2024-11-12T15:12:25,427 DEBUG [RS:0;2b6d221c5cde:34071 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408, exclude list is [], retry=0 2024-11-12T15:12:25,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:25,439 INFO [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408 2024-11-12T15:12:25,440 DEBUG [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:25,501 DEBUG [2b6d221c5cde:43085 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T15:12:25,517 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,525 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2b6d221c5cde,34071,1731424341329, state=OPENING 2024-11-12T15:12:25,549 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T15:12:25,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:25,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:12:25,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T15:12:25,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T15:12:25,562 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T15:12:25,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:25,737 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T15:12:25,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37303, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T15:12:25,751 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T15:12:25,751 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-12T15:12:25,752 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-12T15:12:25,752 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:12:25,752 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T15:12:25,756 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C34071%2C1731424341329.meta, suffix=.meta, logDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329, archiveDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs, maxLogs=32 2024-11-12T15:12:25,777 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta, exclude list is [], retry=0 2024-11-12T15:12:25,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:25,785 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta 2024-11-12T15:12:25,786 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:25,786 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:25,792 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:25,793 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T15:12:25,797 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T15:12:25,797 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T15:12:25,802 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T15:12:25,803 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:25,803 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T15:12:25,803 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T15:12:25,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T15:12:25,809 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T15:12:25,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T15:12:25,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T15:12:25,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T15:12:25,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T15:12:25,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T15:12:25,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T15:12:25,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:25,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T15:12:25,822 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T15:12:25,823 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740 2024-11-12T15:12:25,827 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740 2024-11-12T15:12:25,830 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T15:12:25,830 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T15:12:25,831 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T15:12:25,835 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T15:12:25,838 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10428840320, jitterRate=-0.028738558292388916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T15:12:25,838 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T15:12:25,840 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731424345804Writing region info on filesystem at 1731424345804Initializing all the Stores at 1731424345806 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424345806Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424345807 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424345807Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731424345807Cleaning up temporary data from old regions at 1731424345830 (+23 ms)Running coprocessor post-open hooks at 1731424345838 (+8 ms)Region opened successfully at 1731424345840 (+2 ms) 2024-11-12T15:12:25,849 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731424345730 2024-11-12T15:12:25,862 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T15:12:25,862 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T15:12:25,864 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,867 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2b6d221c5cde,34071,1731424341329, state=OPEN 2024-11-12T15:12:25,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-12T15:12:25,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-12T15:12:25,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T15:12:25,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-12T15:12:25,918 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:25,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T15:12:25,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2b6d221c5cde,34071,1731424341329 in 355 msec 2024-11-12T15:12:25,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T15:12:25,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 843 msec 2024-11-12T15:12:25,936 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T15:12:25,936 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T15:12:25,956 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:25,957 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:25,983 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:25,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41459, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:26,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1830 sec 2024-11-12T15:12:26,016 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731424346016, completionTime=-1 2024-11-12T15:12:26,020 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T15:12:26,020 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T15:12:26,060 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-12T15:12:26,060 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731424406060 2024-11-12T15:12:26,060 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731424466060 2024-11-12T15:12:26,060 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 40 msec 2024-11-12T15:12:26,063 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,43085,1731424339322-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,063 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,43085,1731424339322-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,064 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,43085,1731424339322-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,066 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2b6d221c5cde:43085, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,066 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,067 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,074 DEBUG [master/2b6d221c5cde:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T15:12:26,110 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.465sec 2024-11-12T15:12:26,111 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T15:12:26,113 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T15:12:26,114 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T15:12:26,115 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T15:12:26,115 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T15:12:26,116 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,43085,1731424339322-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T15:12:26,117 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,43085,1731424339322-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T15:12:26,127 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T15:12:26,128 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T15:12:26,128 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,43085,1731424339322-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T15:12:26,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73dc9676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:26,175 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T15:12:26,175 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T15:12:26,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:26,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:26,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:26,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:26,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57edd2d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:26,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:26,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:26,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:26,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:26,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f57d422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:26,193 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:26,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:26,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:26,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:26,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:26,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-12T15:12:26,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/test.cache.data in system properties and HBase conf 2024-11-12T15:12:26,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T15:12:26,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir in system properties and HBase conf 2024-11-12T15:12:26,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T15:12:26,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/nfs.dump.dir in system properties and HBase conf 2024-11-12T15:12:26,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir in system properties and HBase conf 2024-11-12T15:12:26,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T15:12:26,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T15:12:26,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T15:12:26,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741835_1011 (size=592039) 2024-11-12T15:12:26,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741836_1012 (size=1663647) 2024-11-12T15:12:28,073 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:28,158 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:28,369 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T15:12:28,369 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-12T15:12:28,370 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T15:12:28,388 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:28,388 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:28,388 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T15:12:28,391 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26c56adc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:28,391 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a908e18{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T15:12:28,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:28,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:28,394 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T15:12:28,399 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:28,409 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11a358cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:28,410 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c57c6a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T15:12:28,542 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-11-12T15:12:28,543 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-11-12T15:12:28,543 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-12T15:12:28,545 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-12T15:12:28,611 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:28,698 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:29,030 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-11-12T15:12:29,059 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a9ddefe{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir/jetty-localhost-37199-hadoop-yarn-common-3_4_1_jar-_-any-8028638139412139304/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-12T15:12:29,059 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6fc8b3aa{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir/jetty-localhost-37085-hadoop-yarn-common-3_4_1_jar-_-any-11253720522533342865/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-12T15:12:29,060 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@37e8ed89{HTTP/1.1, (http/1.1)}{localhost:37085} 2024-11-12T15:12:29,060 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e5e639{HTTP/1.1, (http/1.1)}{localhost:37199} 2024-11-12T15:12:29,060 INFO [Time-limited test {}] server.Server(415): Started @18240ms 2024-11-12T15:12:29,060 INFO [Thread-251 {}] server.Server(415): Started @18240ms 2024-11-12T15:12:29,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741837_1013 (size=5) 2024-11-12T15:12:29,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T15:12:29,792 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:29,801 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:12:29,801 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-12T15:12:29,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T15:12:29,802 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T15:12:29,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T15:12:29,802 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:30,107 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-12T15:12:30,113 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:30,144 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-12T15:12:30,144 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T15:12:30,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:30,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:30,156 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T15:12:30,160 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:30,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61534992{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:30,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fac336c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T15:12:30,215 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-12T15:12:30,216 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-12T15:12:30,216 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-12T15:12:30,216 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-12T15:12:30,226 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:30,234 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:30,354 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:30,366 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6449e4bf{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir/jetty-localhost-39551-hadoop-yarn-common-3_4_1_jar-_-any-5782454639084488467/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T15:12:30,367 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b8b49f{HTTP/1.1, (http/1.1)}{localhost:39551} 2024-11-12T15:12:30,367 INFO [Time-limited test {}] server.Server(415): Started @19547ms 2024-11-12T15:12:30,534 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-12T15:12:30,536 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:30,552 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-12T15:12:30,553 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T15:12:30,559 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T15:12:30,559 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T15:12:30,559 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T15:12:30,561 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T15:12:30,562 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2788ebc5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,AVAILABLE} 2024-11-12T15:12:30,563 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4db04e37{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-12T15:12:30,624 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-12T15:12:30,624 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-12T15:12:30,624 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-12T15:12:30,624 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-12T15:12:30,631 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:30,635 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:30,721 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-12T15:12:30,727 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@373f13ac{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/java.io.tmpdir/jetty-localhost-39761-hadoop-yarn-common-3_4_1_jar-_-any-4203305761875377673/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T15:12:30,728 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5310ce69{HTTP/1.1, (http/1.1)}{localhost:39761} 2024-11-12T15:12:30,728 INFO [Time-limited test {}] server.Server(415): Started @19908ms 2024-11-12T15:12:30,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-12T15:12:30,753 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:33269/backupUT 2024-11-12T15:12:30,757 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T15:12:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77746083 2024-11-12T15:12:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T15:12:30,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T15:12:30,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns1'} 2024-11-12T15:12:30,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-12T15:12:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T15:12:30,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 23 msec 2024-11-12T15:12:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T15:12:30,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-12T15:12:30,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns2'} 2024-11-12T15:12:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-12T15:12:30,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-12T15:12:30,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-11-12T15:12:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-12T15:12:31,004 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-12T15:12:31,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns3'} 2024-11-12T15:12:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-12T15:12:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-12T15:12:31,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 13 msec 2024-11-12T15:12:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-12T15:12:31,114 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-12T15:12:31,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns4'} 2024-11-12T15:12:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-12T15:12:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T15:12:31,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 15 msec 2024-11-12T15:12:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T15:12:31,225 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-12T15:12:31,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test-1731424350753', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1731424350753 2024-11-12T15:12:31,244 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1731424350753 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:12:31,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test-1731424350753" procId is: 8 2024-11-12T15:12:31,247 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T15:12:31,250 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1731424350753 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:12:31,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741838_1014 (size=367) 2024-11-12T15:12:31,265 INFO [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e4f31acd4066dc4cd42e4f7b965c9073, NAME => 'test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1731424350753', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:31,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741839_1015 (size=53) 2024-11-12T15:12:31,275 DEBUG [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(898): Instantiated test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:31,275 DEBUG [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1722): Closing e4f31acd4066dc4cd42e4f7b965c9073, disabling compactions & flushes 2024-11-12T15:12:31,275 INFO [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1755): Closing region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,276 DEBUG [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,276 DEBUG [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. after waiting 0 ms 2024-11-12T15:12:31,276 DEBUG [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,276 INFO [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1973): Closed test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,276 DEBUG [RegionOpenAndInit-test-1731424350753-pool-0 {}] regionserver.HRegion(1676): Region close journal for e4f31acd4066dc4cd42e4f7b965c9073: Waiting for close lock at 1731424351275Disabling compacts and flushes for region at 1731424351275Disabling writes for close at 1731424351276 (+1 ms)Writing region close event to WAL at 1731424351276Closed at 1731424351276 2024-11-12T15:12:31,278 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1731424350753 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:12:31,284 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731424351278"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424351278"}]},"ts":"1731424351278"} 2024-11-12T15:12:31,289 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:12:31,291 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1731424350753 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:12:31,294 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1731424350753","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424351291"}]},"ts":"1731424351291"} 2024-11-12T15:12:31,300 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1731424350753, state=ENABLING in hbase:meta 2024-11-12T15:12:31,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, ASSIGN}] 2024-11-12T15:12:31,305 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, ASSIGN 2024-11-12T15:12:31,307 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, ASSIGN; state=OFFLINE, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=false 2024-11-12T15:12:31,320 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T15:12:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T15:12:31,381 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T15:12:31,459 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:31,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, ASSIGN because future has completed 2024-11-12T15:12:31,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T15:12:31,649 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,649 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T15:12:31,650 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 2b6d221c5cde,34071,1731424341329.regiongroup-1 2024-11-12T15:12:31,653 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1, suffix=, logDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329, archiveDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs, maxLogs=32 2024-11-12T15:12:31,668 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424351655, exclude list is [], retry=0 2024-11-12T15:12:31,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:31,675 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424351655 2024-11-12T15:12:31,675 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:31,676 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => e4f31acd4066dc4cd42e4f7b965c9073, NAME => 'test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:31,676 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:31,676 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731424350753 e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,677 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:31,677 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,677 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,680 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,682 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName f 2024-11-12T15:12:31,682 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:31,683 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:31,684 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,686 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,687 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,688 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,688 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,691 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,695 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:31,696 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened e4f31acd4066dc4cd42e4f7b965c9073; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11617440960, jitterRate=0.08195850253105164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T15:12:31,696 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:31,697 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for e4f31acd4066dc4cd42e4f7b965c9073: Running coprocessor pre-open hook at 1731424351677Writing region info on filesystem at 1731424351677Initializing all the Stores at 1731424351679 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424351679Cleaning up temporary data from old regions at 1731424351688 (+9 ms)Running coprocessor post-open hooks at 1731424351696 (+8 ms)Region opened successfully at 1731424351697 (+1 ms) 2024-11-12T15:12:31,699 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., pid=10, masterSystemTime=1731424351618 2024-11-12T15:12:31,703 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,703 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:31,704 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:31,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:31,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T15:12:31,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 in 246 msec 2024-11-12T15:12:31,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-12T15:12:31,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, ASSIGN in 413 msec 2024-11-12T15:12:31,721 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1731424350753 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:12:31,722 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1731424350753","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424351721"}]},"ts":"1731424351721"} 2024-11-12T15:12:31,725 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1731424350753, state=ENABLED in hbase:meta 2024-11-12T15:12:31,726 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1731424350753 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:12:31,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1731424350753 in 489 msec 2024-11-12T15:12:31,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-12T15:12:31,875 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1731424350753 completed 2024-11-12T15:12:31,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67655647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:31,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:31,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:31,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7513ac90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:31,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:31,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:31,882 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b46498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:31,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:31,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731424350753', row='row0', locateType=CURRENT is [region=test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:12:31,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] regionserver.HRegion(8528): writing data to region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. with WAL disabled. Data may be lost in the event of a crash. 2024-11-12T15:12:32,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:test-17314243507531', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17314243507531 2024-11-12T15:12:32,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17314243507531 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:12:32,096 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:32,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "test-17314243507531" procId is: 11 2024-11-12T15:12:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T15:12:32,098 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17314243507531 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:12:32,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741841_1017 (size=364) 2024-11-12T15:12:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T15:12:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T15:12:32,511 INFO [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0e9acca221b224195ce9cae6845295fe, NAME => 'ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17314243507531', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:32,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741842_1018 (size=50) 2024-11-12T15:12:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T15:12:32,921 DEBUG [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:32,922 DEBUG [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1722): Closing 0e9acca221b224195ce9cae6845295fe, disabling compactions & flushes 2024-11-12T15:12:32,922 INFO [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:32,922 DEBUG [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:32,922 DEBUG [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. after waiting 0 ms 2024-11-12T15:12:32,922 DEBUG [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:32,922 INFO [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:32,922 DEBUG [RegionOpenAndInit-ns2:test-17314243507531-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0e9acca221b224195ce9cae6845295fe: Waiting for close lock at 1731424352922Disabling compacts and flushes for region at 1731424352922Disabling writes for close at 1731424352922Writing region close event to WAL at 1731424352922Closed at 1731424352922 2024-11-12T15:12:32,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17314243507531 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:12:32,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731424352924"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424352924"}]},"ts":"1731424352924"} 2024-11-12T15:12:32,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:12:32,930 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17314243507531 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:12:32,931 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17314243507531","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424352930"}]},"ts":"1731424352930"} 2024-11-12T15:12:32,934 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17314243507531, state=ENABLING in hbase:meta 2024-11-12T15:12:32,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17314243507531, region=0e9acca221b224195ce9cae6845295fe, ASSIGN}] 2024-11-12T15:12:32,936 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17314243507531, region=0e9acca221b224195ce9cae6845295fe, ASSIGN 2024-11-12T15:12:32,938 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17314243507531, region=0e9acca221b224195ce9cae6845295fe, ASSIGN; state=OFFLINE, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=false 2024-11-12T15:12:33,089 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0e9acca221b224195ce9cae6845295fe, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:33,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17314243507531, region=0e9acca221b224195ce9cae6845295fe, ASSIGN because future has completed 2024-11-12T15:12:33,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e9acca221b224195ce9cae6845295fe, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T15:12:33,252 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:33,253 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e9acca221b224195ce9cae6845295fe, NAME => 'ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:33,253 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:33,253 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17314243507531 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,253 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:33,254 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,254 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,256 INFO [StoreOpener-0e9acca221b224195ce9cae6845295fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,259 INFO [StoreOpener-0e9acca221b224195ce9cae6845295fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e9acca221b224195ce9cae6845295fe columnFamilyName f 2024-11-12T15:12:33,259 DEBUG [StoreOpener-0e9acca221b224195ce9cae6845295fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:33,260 INFO [StoreOpener-0e9acca221b224195ce9cae6845295fe-1 {}] regionserver.HStore(327): Store=0e9acca221b224195ce9cae6845295fe/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:33,261 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,262 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,264 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,265 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,265 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,269 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,273 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:33,274 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 0e9acca221b224195ce9cae6845295fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9672280640, jitterRate=-0.09919866919517517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T15:12:33,274 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:33,275 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 0e9acca221b224195ce9cae6845295fe: Running coprocessor pre-open hook at 1731424353254Writing region info on filesystem at 1731424353254Initializing all the Stores at 1731424353256 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424353256Cleaning up temporary data from old regions at 1731424353265 (+9 ms)Running coprocessor post-open hooks at 1731424353274 (+9 ms)Region opened successfully at 1731424353275 (+1 ms) 2024-11-12T15:12:33,276 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe., pid=13, masterSystemTime=1731424353247 2024-11-12T15:12:33,280 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:33,280 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:33,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0e9acca221b224195ce9cae6845295fe, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:33,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e9acca221b224195ce9cae6845295fe, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:33,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-12T15:12:33,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e9acca221b224195ce9cae6845295fe, server=2b6d221c5cde,34071,1731424341329 in 193 msec 2024-11-12T15:12:33,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T15:12:33,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17314243507531, region=0e9acca221b224195ce9cae6845295fe, ASSIGN in 357 msec 2024-11-12T15:12:33,297 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17314243507531 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:12:33,297 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17314243507531","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424353297"}]},"ts":"1731424353297"} 2024-11-12T15:12:33,300 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17314243507531, state=ENABLED in hbase:meta 2024-11-12T15:12:33,302 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17314243507531 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:12:33,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17314243507531 in 1.2100 sec 2024-11-12T15:12:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T15:12:34,255 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17314243507531 completed 2024-11-12T15:12:34,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17314243507531', row='row0', locateType=CURRENT is [region=ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:12:34,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] regionserver.HRegion(8528): writing data to region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. with WAL disabled. Data may be lost in the event of a crash. 2024-11-12T15:12:34,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns3:test-17314243507532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17314243507532 2024-11-12T15:12:34,517 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17314243507532 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:12:34,517 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:34,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns3" qualifier: "test-17314243507532" procId is: 14 2024-11-12T15:12:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T15:12:34,522 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17314243507532 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:12:34,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741843_1019 (size=390) 2024-11-12T15:12:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T15:12:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T15:12:34,935 INFO [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa6d9848b308420055c29add59789661, NAME => 'ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17314243507532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:34,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741844_1020 (size=50) 2024-11-12T15:12:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T15:12:35,347 DEBUG [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:35,347 DEBUG [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1722): Closing aa6d9848b308420055c29add59789661, disabling compactions & flushes 2024-11-12T15:12:35,347 INFO [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,347 DEBUG [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,347 DEBUG [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. after waiting 0 ms 2024-11-12T15:12:35,347 DEBUG [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,348 INFO [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,348 DEBUG [RegionOpenAndInit-ns3:test-17314243507532-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa6d9848b308420055c29add59789661: Waiting for close lock at 1731424355347Disabling compacts and flushes for region at 1731424355347Disabling writes for close at 1731424355347Writing region close event to WAL at 1731424355347Closed at 1731424355348 (+1 ms) 2024-11-12T15:12:35,350 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17314243507532 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:12:35,350 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731424355350"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424355350"}]},"ts":"1731424355350"} 2024-11-12T15:12:35,354 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:12:35,355 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17314243507532 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:12:35,356 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17314243507532","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424355356"}]},"ts":"1731424355356"} 2024-11-12T15:12:35,359 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17314243507532, state=ENABLING in hbase:meta 2024-11-12T15:12:35,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17314243507532, region=aa6d9848b308420055c29add59789661, ASSIGN}] 2024-11-12T15:12:35,362 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17314243507532, region=aa6d9848b308420055c29add59789661, ASSIGN 2024-11-12T15:12:35,363 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17314243507532, region=aa6d9848b308420055c29add59789661, ASSIGN; state=OFFLINE, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=false 2024-11-12T15:12:35,514 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=aa6d9848b308420055c29add59789661, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:35,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17314243507532, region=aa6d9848b308420055c29add59789661, ASSIGN because future has completed 2024-11-12T15:12:35,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa6d9848b308420055c29add59789661, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T15:12:35,678 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,678 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => aa6d9848b308420055c29add59789661, NAME => 'ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:35,678 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:35,679 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17314243507532 aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,679 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:35,679 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,679 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,682 INFO [StoreOpener-aa6d9848b308420055c29add59789661-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,684 INFO [StoreOpener-aa6d9848b308420055c29add59789661-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa6d9848b308420055c29add59789661 columnFamilyName f 2024-11-12T15:12:35,684 DEBUG [StoreOpener-aa6d9848b308420055c29add59789661-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:35,685 INFO [StoreOpener-aa6d9848b308420055c29add59789661-1 {}] regionserver.HStore(327): Store=aa6d9848b308420055c29add59789661/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:35,685 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,687 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns3/test-17314243507532/aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,688 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns3/test-17314243507532/aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,688 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,688 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,692 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,696 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns3/test-17314243507532/aa6d9848b308420055c29add59789661/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:35,697 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened aa6d9848b308420055c29add59789661; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11465842080, jitterRate=0.06783975660800934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T15:12:35,697 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa6d9848b308420055c29add59789661 2024-11-12T15:12:35,698 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for aa6d9848b308420055c29add59789661: Running coprocessor pre-open hook at 1731424355679Writing region info on filesystem at 1731424355679Initializing all the Stores at 1731424355681 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424355681Cleaning up temporary data from old regions at 1731424355688 (+7 ms)Running coprocessor post-open hooks at 1731424355697 (+9 ms)Region opened successfully at 1731424355698 (+1 ms) 2024-11-12T15:12:35,700 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661., pid=16, masterSystemTime=1731424355672 2024-11-12T15:12:35,703 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,703 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:12:35,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=aa6d9848b308420055c29add59789661, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:35,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa6d9848b308420055c29add59789661, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:35,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T15:12:35,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure aa6d9848b308420055c29add59789661, server=2b6d221c5cde,34071,1731424341329 in 192 msec 2024-11-12T15:12:35,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-12T15:12:35,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17314243507532, region=aa6d9848b308420055c29add59789661, ASSIGN in 356 msec 2024-11-12T15:12:35,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17314243507532 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:12:35,721 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17314243507532","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424355721"}]},"ts":"1731424355721"} 2024-11-12T15:12:35,724 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17314243507532, state=ENABLED in hbase:meta 2024-11-12T15:12:35,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17314243507532 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:12:35,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17314243507532 in 1.2130 sec 2024-11-12T15:12:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-12T15:12:36,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17314243507532 completed 2024-11-12T15:12:36,665 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17314243507532 get assigned. Timeout = 60000ms 2024-11-12T15:12:36,667 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:12:36,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17314243507532 assigned to meta. Checking AM states. 2024-11-12T15:12:36,673 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:12:36,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17314243507532 assigned. 2024-11-12T15:12:36,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns4:test-17314243507533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17314243507533 2024-11-12T15:12:36,680 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17314243507533 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:12:36,680 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:36,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns4" qualifier: "test-17314243507533" procId is: 17 2024-11-12T15:12:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T15:12:36,682 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17314243507533 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:12:36,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741845_1021 (size=390) 2024-11-12T15:12:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T15:12:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T15:12:37,103 INFO [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b64e86ccc223899acbc0b981c82d38b4, NAME => 'ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17314243507533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:37,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741846_1022 (size=50) 2024-11-12T15:12:37,180 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T15:12:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T15:12:37,383 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17314243507532' 2024-11-12T15:12:37,383 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1731424350753' 2024-11-12T15:12:37,384 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17314243507531' 2024-11-12T15:12:37,516 DEBUG [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:37,516 DEBUG [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1722): Closing b64e86ccc223899acbc0b981c82d38b4, disabling compactions & flushes 2024-11-12T15:12:37,516 INFO [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,517 DEBUG [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,517 DEBUG [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. after waiting 0 ms 2024-11-12T15:12:37,517 DEBUG [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,517 INFO [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,517 DEBUG [RegionOpenAndInit-ns4:test-17314243507533-pool-0 {}] regionserver.HRegion(1676): Region close journal for b64e86ccc223899acbc0b981c82d38b4: Waiting for close lock at 1731424357516Disabling compacts and flushes for region at 1731424357516Disabling writes for close at 1731424357517 (+1 ms)Writing region close event to WAL at 1731424357517Closed at 1731424357517 2024-11-12T15:12:37,525 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17314243507533 execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:12:37,526 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1731424357525"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424357525"}]},"ts":"1731424357525"} 2024-11-12T15:12:37,532 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:12:37,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17314243507533 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:12:37,535 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17314243507533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424357534"}]},"ts":"1731424357534"} 2024-11-12T15:12:37,538 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17314243507533, state=ENABLING in hbase:meta 2024-11-12T15:12:37,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17314243507533, region=b64e86ccc223899acbc0b981c82d38b4, ASSIGN}] 2024-11-12T15:12:37,540 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17314243507533, region=b64e86ccc223899acbc0b981c82d38b4, ASSIGN 2024-11-12T15:12:37,542 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17314243507533, region=b64e86ccc223899acbc0b981c82d38b4, ASSIGN; state=OFFLINE, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=false 2024-11-12T15:12:37,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b64e86ccc223899acbc0b981c82d38b4, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:37,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17314243507533, region=b64e86ccc223899acbc0b981c82d38b4, ASSIGN because future has completed 2024-11-12T15:12:37,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b64e86ccc223899acbc0b981c82d38b4, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T15:12:37,856 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,856 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => b64e86ccc223899acbc0b981c82d38b4, NAME => 'ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:37,857 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:37,857 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17314243507533 b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,857 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:37,857 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,857 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,859 INFO [StoreOpener-b64e86ccc223899acbc0b981c82d38b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,862 INFO [StoreOpener-b64e86ccc223899acbc0b981c82d38b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b64e86ccc223899acbc0b981c82d38b4 columnFamilyName f 2024-11-12T15:12:37,862 DEBUG [StoreOpener-b64e86ccc223899acbc0b981c82d38b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:37,863 INFO [StoreOpener-b64e86ccc223899acbc0b981c82d38b4-1 {}] regionserver.HStore(327): Store=b64e86ccc223899acbc0b981c82d38b4/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:37,863 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,864 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns4/test-17314243507533/b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,865 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns4/test-17314243507533/b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,866 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,866 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,868 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,871 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns4/test-17314243507533/b64e86ccc223899acbc0b981c82d38b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:37,872 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened b64e86ccc223899acbc0b981c82d38b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11390302240, jitterRate=0.06080456078052521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T15:12:37,872 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:12:37,873 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for b64e86ccc223899acbc0b981c82d38b4: Running coprocessor pre-open hook at 1731424357857Writing region info on filesystem at 1731424357857Initializing all the Stores at 1731424357858 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424357859 (+1 ms)Cleaning up temporary data from old regions at 1731424357866 (+7 ms)Running coprocessor post-open hooks at 1731424357872 (+6 ms)Region opened successfully at 1731424357873 (+1 ms) 2024-11-12T15:12:37,874 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4., pid=19, masterSystemTime=1731424357851 2024-11-12T15:12:37,877 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,877 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:12:37,878 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b64e86ccc223899acbc0b981c82d38b4, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:37,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b64e86ccc223899acbc0b981c82d38b4, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:37,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-12T15:12:37,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure b64e86ccc223899acbc0b981c82d38b4, server=2b6d221c5cde,34071,1731424341329 in 186 msec 2024-11-12T15:12:37,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T15:12:37,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17314243507533, region=b64e86ccc223899acbc0b981c82d38b4, ASSIGN in 349 msec 2024-11-12T15:12:37,892 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17314243507533 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:12:37,892 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17314243507533","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424357892"}]},"ts":"1731424357892"} 2024-11-12T15:12:37,895 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17314243507533, state=ENABLED in hbase:meta 2024-11-12T15:12:37,896 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17314243507533 execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:12:37,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17314243507533 in 1.2200 sec 2024-11-12T15:12:38,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T15:12:38,835 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17314243507533 completed 2024-11-12T15:12:38,835 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17314243507533 get assigned. Timeout = 60000ms 2024-11-12T15:12:38,835 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:12:38,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17314243507533 assigned to meta. Checking AM states. 2024-11-12T15:12:38,840 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:12:38,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17314243507533 assigned. 2024-11-12T15:12:38,840 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:12:38,840 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:12:38,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:38,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:38,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:12:38,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=9, AvailableMemoryMB=4708 2024-11-12T15:12:38,876 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-12T15:12:38,878 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-12T15:12:38,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1731424350753 from 'test-1731424350753', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1731424350753', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:38,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1731424350753 2024-11-12T15:12:38,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741847_1023 (size=908) 2024-11-12T15:12:38,914 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000001.367 2024-11-12T15:12:38,914 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000002.908 2024-11-12T15:12:38,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1731424350753}] 2024-11-12T15:12:38,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE}] 2024-11-12T15:12:38,936 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE 2024-11-12T15:12:38,939 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=CLOSING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:38,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE because future has completed 2024-11-12T15:12:38,943 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T15:12:38,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:39,104 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:39,104 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T15:12:39,105 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing e4f31acd4066dc4cd42e4f7b965c9073, disabling compactions & flushes 2024-11-12T15:12:39,105 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:39,105 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:39,105 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. after waiting 0 ms 2024-11-12T15:12:39,105 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:39,108 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing e4f31acd4066dc4cd42e4f7b965c9073 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-12T15:12:39,184 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f/245e1c5baac741b5b8cbe8ed5b46b824 is 37, key is row10/f:q1/1731424351919/Put/seqid=0 2024-11-12T15:12:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741848_1024 (size=8413) 2024-11-12T15:12:39,595 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f/245e1c5baac741b5b8cbe8ed5b46b824 2024-11-12T15:12:39,656 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f/245e1c5baac741b5b8cbe8ed5b46b824 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 2024-11-12T15:12:39,667 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824, entries=99, sequenceid=103, filesize=8.2 K 2024-11-12T15:12:39,674 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for e4f31acd4066dc4cd42e4f7b965c9073 in 565ms, sequenceid=103, compaction requested=false 2024-11-12T15:12:39,682 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-12T15:12:39,685 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:12:39,685 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:39,685 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for e4f31acd4066dc4cd42e4f7b965c9073: Waiting for close lock at 1731424359105Running coprocessor pre-close hooks at 1731424359105Disabling compacts and flushes for region at 1731424359105Disabling writes for close at 1731424359105Obtaining lock to block concurrent updates at 1731424359108 (+3 ms)Preparing flush snapshotting stores in e4f31acd4066dc4cd42e4f7b965c9073 at 1731424359109 (+1 ms)Finished memstore snapshotting test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1731424359116 (+7 ms)Flushing stores of test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. at 1731424359118 (+2 ms)Flushing e4f31acd4066dc4cd42e4f7b965c9073/f: creating writer at 1731424359120 (+2 ms)Flushing e4f31acd4066dc4cd42e4f7b965c9073/f: appending metadata at 1731424359171 (+51 ms)Flushing e4f31acd4066dc4cd42e4f7b965c9073/f: closing flushed file at 1731424359174 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7619f19e: reopening flushed file at 1731424359654 (+480 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for e4f31acd4066dc4cd42e4f7b965c9073 in 565ms, sequenceid=103, compaction requested=false at 1731424359674 (+20 ms)Writing region close event to WAL at 1731424359676 (+2 ms)Running coprocessor post-close hooks at 1731424359683 (+7 ms)Closed at 1731424359685 (+2 ms) 2024-11-12T15:12:39,685 WARN [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: e4f31acd4066dc4cd42e4f7b965c9073 to self. 2024-11-12T15:12:39,689 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:39,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=CLOSED 2024-11-12T15:12:39,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:39,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-12T15:12:39,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 in 751 msec 2024-11-12T15:12:39,700 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE; state=CLOSED, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=true 2024-11-12T15:12:39,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17314243507531 2024-11-12T15:12:39,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17314243507531 Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:39,792 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17314243507532 2024-11-12T15:12:39,792 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17314243507532 Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:39,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17314243507533 2024-11-12T15:12:39,793 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17314243507533 Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:39,851 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:39,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE because future has completed 2024-11-12T15:12:39,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:40,013 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:40,013 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => e4f31acd4066dc4cd42e4f7b965c9073, NAME => 'test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:40,013 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:40,013 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731424350753 e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,013 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:40,014 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,014 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,016 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,017 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName f 2024-11-12T15:12:40,017 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:40,028 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 2024-11-12T15:12:40,029 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:40,029 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,031 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName f3 2024-11-12T15:12:40,031 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:40,032 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:40,033 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,041 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName mob 2024-11-12T15:12:40,046 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:40,047 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:40,048 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,049 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,051 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,053 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,053 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,055 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1731424350753 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T15:12:40,057 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,059 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened e4f31acd4066dc4cd42e4f7b965c9073; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10318683520, jitterRate=-0.03899770975112915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T15:12:40,060 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:40,060 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for e4f31acd4066dc4cd42e4f7b965c9073: Running coprocessor pre-open hook at 1731424360014Writing region info on filesystem at 1731424360014Initializing all the Stores at 1731424360015 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424360015Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424360015Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424360015Cleaning up temporary data from old regions at 1731424360053 (+38 ms)Running coprocessor post-open hooks at 1731424360060 (+7 ms)Region opened successfully at 1731424360060 2024-11-12T15:12:40,062 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., pid=24, masterSystemTime=1731424360008 2024-11-12T15:12:40,065 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:40,065 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:40,066 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=OPEN, openSeqNum=107, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:40,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:40,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-12T15:12:40,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 in 216 msec 2024-11-12T15:12:40,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T15:12:40,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE in 1.1400 sec 2024-11-12T15:12:40,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-12T15:12:40,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1731424350753 in 1.1590 sec 2024-11-12T15:12:40,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731424350753 in 1.1990 sec 2024-11-12T15:12:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-12T15:12:40,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1731424350753 completed 2024-11-12T15:12:40,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179a5af4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:40,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:40,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:40,095 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:40,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:40,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:40,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47275e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:40,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:40,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:40,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:40,098 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:40,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba2131a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:40,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:40,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:40,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:40,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:40,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731424350753', row='row-test-1731424350753-3-0', locateType=CURRENT is [region=test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., hostname=2b6d221c5cde,34071,1731424341329, seqNum=107] 2024-11-12T15:12:40,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T15:12:40,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:40,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f90d348 2024-11-12T15:12:40,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T15:12:40,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T15:12:40,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'backup'} 2024-11-12T15:12:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-12T15:12:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T15:12:40,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 19 msec 2024-11-12T15:12:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T15:12:40,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-12T15:12:40,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-12T15:12:40,334 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:12:40,335 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:40,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-12T15:12:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T15:12:40,337 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:12:40,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741849_1025 (size=635) 2024-11-12T15:12:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T15:12:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T15:12:40,749 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9ce3fb3bd1b279fea0145fdedac19450, NAME => 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:40,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741850_1026 (size=40) 2024-11-12T15:12:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T15:12:41,159 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:41,159 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 9ce3fb3bd1b279fea0145fdedac19450, disabling compactions & flushes 2024-11-12T15:12:41,159 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,159 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,159 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. after waiting 0 ms 2024-11-12T15:12:41,159 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,159 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,159 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9ce3fb3bd1b279fea0145fdedac19450: Waiting for close lock at 1731424361159Disabling compacts and flushes for region at 1731424361159Disabling writes for close at 1731424361159Writing region close event to WAL at 1731424361159Closed at 1731424361159 2024-11-12T15:12:41,161 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:12:41,162 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1731424361161"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424361161"}]},"ts":"1731424361161"} 2024-11-12T15:12:41,165 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:12:41,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:12:41,167 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424361167"}]},"ts":"1731424361167"} 2024-11-12T15:12:41,170 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-12T15:12:41,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN}] 2024-11-12T15:12:41,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN 2024-11-12T15:12:41,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN; state=OFFLINE, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=false 2024-11-12T15:12:41,325 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9ce3fb3bd1b279fea0145fdedac19450, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:41,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN because future has completed 2024-11-12T15:12:41,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T15:12:41,487 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,488 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ce3fb3bd1b279fea0145fdedac19450, NAME => 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:41,488 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:41,488 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,488 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:41,489 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,489 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,491 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,494 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ce3fb3bd1b279fea0145fdedac19450 columnFamilyName meta 2024-11-12T15:12:41,494 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:41,495 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(327): Store=9ce3fb3bd1b279fea0145fdedac19450/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:41,496 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,498 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ce3fb3bd1b279fea0145fdedac19450 columnFamilyName session 2024-11-12T15:12:41,498 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:41,500 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(327): Store=9ce3fb3bd1b279fea0145fdedac19450/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:41,500 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,505 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,506 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,507 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,507 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,508 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T15:12:41,514 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,518 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:41,519 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 9ce3fb3bd1b279fea0145fdedac19450; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10646088160, jitterRate=-0.008505776524543762}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T15:12:41,519 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:41,520 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 9ce3fb3bd1b279fea0145fdedac19450: Running coprocessor pre-open hook at 1731424361489Writing region info on filesystem at 1731424361489Initializing all the Stores at 1731424361491 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424361491Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424361491Cleaning up temporary data from old regions at 1731424361507 (+16 ms)Running coprocessor post-open hooks at 1731424361519 (+12 ms)Region opened successfully at 1731424361520 (+1 ms) 2024-11-12T15:12:41,521 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., pid=28, masterSystemTime=1731424361482 2024-11-12T15:12:41,524 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,525 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:41,526 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9ce3fb3bd1b279fea0145fdedac19450, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:41,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:41,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T15:12:41,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329 in 202 msec 2024-11-12T15:12:41,542 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:12:41,542 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424361542"}]},"ts":"1731424361542"} 2024-11-12T15:12:41,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-12T15:12:41,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN in 365 msec 2024-11-12T15:12:41,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-12T15:12:41,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:12:41,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.2160 sec 2024-11-12T15:12:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T15:12:42,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-12T15:12:42,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:12:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-12T15:12:42,515 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T15:12:42,515 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:42,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-12T15:12:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T15:12:42,517 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T15:12:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741851_1027 (size=640) 2024-11-12T15:12:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T15:12:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T15:12:42,928 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb8c7f767a06180d5cc67ec654094c61, NAME => 'backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:42,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741852_1028 (size=45) 2024-11-12T15:12:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T15:12:43,338 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:43,338 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing fb8c7f767a06180d5cc67ec654094c61, disabling compactions & flushes 2024-11-12T15:12:43,338 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,338 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,338 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. after waiting 0 ms 2024-11-12T15:12:43,338 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,338 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,338 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb8c7f767a06180d5cc67ec654094c61: Waiting for close lock at 1731424363338Disabling compacts and flushes for region at 1731424363338Disabling writes for close at 1731424363338Writing region close event to WAL at 1731424363338Closed at 1731424363338 2024-11-12T15:12:43,340 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T15:12:43,341 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731424363340"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424363340"}]},"ts":"1731424363340"} 2024-11-12T15:12:43,344 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:12:43,345 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T15:12:43,346 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424363346"}]},"ts":"1731424363346"} 2024-11-12T15:12:43,349 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-12T15:12:43,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=fb8c7f767a06180d5cc67ec654094c61, ASSIGN}] 2024-11-12T15:12:43,351 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=fb8c7f767a06180d5cc67ec654094c61, ASSIGN 2024-11-12T15:12:43,353 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=fb8c7f767a06180d5cc67ec654094c61, ASSIGN; state=OFFLINE, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=false 2024-11-12T15:12:43,360 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T15:12:43,403 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17314243507533' 2024-11-12T15:12:43,403 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-12T15:12:43,503 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=fb8c7f767a06180d5cc67ec654094c61, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:43,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=fb8c7f767a06180d5cc67ec654094c61, ASSIGN because future has completed 2024-11-12T15:12:43,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8c7f767a06180d5cc67ec654094c61, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:12:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T15:12:43,675 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,675 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => fb8c7f767a06180d5cc67ec654094c61, NAME => 'backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:12:43,676 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:12:43,676 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,676 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:12:43,676 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,677 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,680 INFO [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,691 INFO [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8c7f767a06180d5cc67ec654094c61 columnFamilyName meta 2024-11-12T15:12:43,691 DEBUG [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:43,692 INFO [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] regionserver.HStore(327): Store=fb8c7f767a06180d5cc67ec654094c61/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:43,692 INFO [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,699 INFO [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8c7f767a06180d5cc67ec654094c61 columnFamilyName session 2024-11-12T15:12:43,699 DEBUG [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:43,701 INFO [StoreOpener-fb8c7f767a06180d5cc67ec654094c61-1 {}] regionserver.HStore(327): Store=fb8c7f767a06180d5cc67ec654094c61/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:12:43,702 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,703 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system_bulk/fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,704 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system_bulk/fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,705 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,705 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,706 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T15:12:43,710 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,726 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system_bulk/fb8c7f767a06180d5cc67ec654094c61/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T15:12:43,728 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened fb8c7f767a06180d5cc67ec654094c61; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9973988000, jitterRate=-0.07109998166561127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T15:12:43,729 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:12:43,730 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for fb8c7f767a06180d5cc67ec654094c61: Running coprocessor pre-open hook at 1731424363677Writing region info on filesystem at 1731424363677Initializing all the Stores at 1731424363679 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424363679Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424363680 (+1 ms)Cleaning up temporary data from old regions at 1731424363705 (+25 ms)Running coprocessor post-open hooks at 1731424363729 (+24 ms)Region opened successfully at 1731424363729 2024-11-12T15:12:43,731 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61., pid=31, masterSystemTime=1731424363661 2024-11-12T15:12:43,740 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,740 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:12:43,747 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=fb8c7f767a06180d5cc67ec654094c61, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:43,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8c7f767a06180d5cc67ec654094c61, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:12:43,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-12T15:12:43,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure fb8c7f767a06180d5cc67ec654094c61, server=2b6d221c5cde,34071,1731424341329 in 273 msec 2024-11-12T15:12:43,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T15:12:43,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=fb8c7f767a06180d5cc67ec654094c61, ASSIGN in 449 msec 2024-11-12T15:12:43,825 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T15:12:43,825 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424363825"}]},"ts":"1731424363825"} 2024-11-12T15:12:43,831 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-12T15:12:43,834 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T15:12:43,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.3220 sec 2024-11-12T15:12:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T15:12:44,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-12T15:12:44,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:12:44,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:12:44,702 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T15:12:44,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:12:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34071 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-12T15:12:44,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:12:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424364745 (current time:1731424364745). 2024-11-12T15:12:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:12:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T15:12:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f2274e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:44,751 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:44,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:44,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:44,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688ce502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:44,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:44,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:44,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:44,761 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:44,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@843141b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:44,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:44,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:44,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:44,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:12:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:12:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:44,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:12:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:12:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:12:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-12T15:12:44,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T15:12:44,812 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:12:44,818 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:12:44,837 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:12:44,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741853_1029 (size=52) 2024-11-12T15:12:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T15:12:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T15:12:45,268 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:12:45,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450}] 2024-11-12T15:12:45,278 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:45,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-12T15:12:45,440 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:45,441 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 9ce3fb3bd1b279fea0145fdedac19450 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-12T15:12:45,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T15:12:45,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/2d1e985d0d7044c1972e4dbf0dba516d is 49, key is activesession:/session:c/1731424364728/Put/seqid=0 2024-11-12T15:12:45,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741854_1030 (size=5049) 2024-11-12T15:12:45,505 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/2d1e985d0d7044c1972e4dbf0dba516d 2024-11-12T15:12:45,532 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/2d1e985d0d7044c1972e4dbf0dba516d as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d 2024-11-12T15:12:45,567 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, entries=1, sequenceid=5, filesize=4.9 K 2024-11-12T15:12:45,583 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 142ms, sequenceid=5, compaction requested=false 2024-11-12T15:12:45,584 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 9ce3fb3bd1b279fea0145fdedac19450: 2024-11-12T15:12:45,584 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. for snapshot_backup_system completed. 2024-11-12T15:12:45,585 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.' region-info for snapshot=snapshot_backup_system 2024-11-12T15:12:45,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:12:45,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T15:12:45,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d] hfiles 2024-11-12T15:12:45,599 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d for snapshot=snapshot_backup_system 2024-11-12T15:12:45,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741855_1031 (size=95) 2024-11-12T15:12:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T15:12:46,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:12:46,038 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-12T15:12:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-12T15:12:46,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:46,042 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:12:46,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T15:12:46,047 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:12:46,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 in 772 msec 2024-11-12T15:12:46,050 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:12:46,051 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:12:46,052 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T15:12:46,055 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T15:12:46,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741856_1032 (size=731) 2024-11-12T15:12:46,516 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:12:46,528 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:12:46,529 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_backup_system 2024-11-12T15:12:46,532 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:12:46,532 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-12T15:12:46,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7310 sec 2024-11-12T15:12:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T15:12:46,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T15:12:46,975 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731424360169 started at 1731424366974. 2024-11-12T15:12:46,993 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731424360169 has been started. 2024-11-12T15:12:47,000 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-12T15:12:47,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-12T15:12:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T15:12:47,011 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T15:12:47,011 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T15:12:47,011 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T15:12:47,011 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T15:12:47,012 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,012 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:12:47,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:12:47,053 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,053 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:12:47,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:12:47,054 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T15:12:47,055 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,055 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T15:12:47,055 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T15:12:47,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:12:47,056 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T15:12:47,059 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T15:12:47,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T15:12:47,061 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T15:12:47,061 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T15:12:47,061 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T15:12:47,061 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T15:12:47,062 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T15:12:47,074 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:12:47,075 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:12:47,075 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,075 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T15:12:47,076 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:12:47,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:12:47,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:12:47,078 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,078 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:12:47,079 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T15:12:47,079 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T15:12:47,079 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@44350da6[Count = 0] remaining members to acquire global barrier 2024-11-12T15:12:47,079 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,093 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,094 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T15:12:47,094 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,094 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T15:12:47,095 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T15:12:47,095 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,105 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731424345408 highest: 1731424351655 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,105 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1:(num 1731424351655) roll requested 2024-11-12T15:12:47,126 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105, exclude list is [], retry=0 2024-11-12T15:12:47,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:47,142 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424351655 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 2024-11-12T15:12:47,143 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:47,143 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424351655 is not closed yet, will try archiving it next time 2024-11-12T15:12:47,143 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.meta:.meta(num 1731424345759) roll requested 2024-11-12T15:12:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741840_1016 (size=4756) 2024-11-12T15:12:47,160 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta, exclude list is [], retry=0 2024-11-12T15:12:47,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:47,171 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta 2024-11-12T15:12:47,172 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:47,172 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta is not closed yet, will try archiving it next time 2024-11-12T15:12:47,172 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0:(num 1731424345408) roll requested 2024-11-12T15:12:47,187 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172, exclude list is [], retry=0 2024-11-12T15:12:47,197 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta not finished, retry = 0 2024-11-12T15:12:47,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741834_1010 (size=12055) 2024-11-12T15:12:47,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:12:47,207 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 2024-11-12T15:12:47,212 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:12:47,212 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408 is not closed yet, will try archiving it next time 2024-11-12T15:12:47,212 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 107 2024-11-12T15:12:47,213 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731424367172 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,213 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T15:12:47,213 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,43085,1731424339322 2024-11-12T15:12:47,214 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b69e3fd 2024-11-12T15:12:47,214 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T15:12:47,217 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-12T15:12:47,218 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:47,218 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:47,218 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:47,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46705, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-12T15:12:47,229 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408 not finished, retry = 0 2024-11-12T15:12:47,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741833_1009 (size=1034) 2024-11-12T15:12:47,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:12:47,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:12:47,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:33269/backupUT\x00', locateType=CURRENT is [region=backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:12:47,302 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T15:12:47,302 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '2b6d221c5cde,34071,1731424341329' in zk 2024-11-12T15:12:47,316 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T15:12:47,316 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:12:47,316 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T15:12:47,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:12:47,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:12:47,321 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:12:47,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:12:47,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:12:47,323 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:12:47,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:12:47,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '2b6d221c5cde,34071,1731424341329': 2024-11-12T15:12:47,331 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '2b6d221c5cde,34071,1731424341329' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T15:12:47,331 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T15:12:47,331 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T15:12:47,331 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T15:12:47,331 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T15:12:47,332 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T15:12:47,337 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424345408 2024-11-12T15:12:47,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:12:47,345 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,345 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,347 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:12:47,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:12:47,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:12:47,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:12:47,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:12:47,355 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:12:47,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:12:47,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:12:47,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,382 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:12:47,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:12:47,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:12:47,401 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:12:47,401 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:12:47,401 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:12:47,401 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:12:47,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:12:47,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:12:47,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T15:12:47,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T15:12:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T15:12:47,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } 2024-11-12T15:12:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424367435 (current time:1731424367435). 2024-11-12T15:12:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:12:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424367433_default_test-1731424350753 VERSION not specified, setting to 2 2024-11-12T15:12:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294cc7b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:47,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:47,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:47,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:47,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b18ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:47,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:47,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:47,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:47,446 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:47,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70ec75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:47,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:47,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:47,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:47,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:12:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:12:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:47,467 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:12:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:12:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } 2024-11-12T15:12:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-12T15:12:47,473 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:12:47,475 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:12:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:47,484 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:12:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741860_1036 (size=84) 2024-11-12T15:12:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:47,919 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:12:47,919 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073}] 2024-11-12T15:12:47,921 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:48,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-12T15:12:48,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:48,075 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing e4f31acd4066dc4cd42e4f7b965c9073 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-12T15:12:48,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f3/cca7d187282e4238a0f131484a6889ae is 58, key is row-test-1731424350753-3-0/f3:q1/1731424360108/Put/seqid=0 2024-11-12T15:12:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:48,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741861_1037 (size=5387) 2024-11-12T15:12:48,231 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T15:12:48,510 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f3/cca7d187282e4238a0f131484a6889ae 2024-11-12T15:12:48,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741862_1038 (size=4469) 2024-11-12T15:12:48,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/dffaf2b995334a6dad0b06c90b71accb is 59, key is row-test-1731424350753-3-0/mob:q1/1731424360151/Put/seqid=0 2024-11-12T15:12:48,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741863_1039 (size=5484) 2024-11-12T15:12:48,597 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/dffaf2b995334a6dad0b06c90b71accb 2024-11-12T15:12:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:48,625 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f3/cca7d187282e4238a0f131484a6889ae as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae 2024-11-12T15:12:48,647 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae, entries=6, sequenceid=121, filesize=5.3 K 2024-11-12T15:12:48,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/dffaf2b995334a6dad0b06c90b71accb as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb 2024-11-12T15:12:48,661 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb, entries=6, sequenceid=121, filesize=5.4 K 2024-11-12T15:12:48,663 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for e4f31acd4066dc4cd42e4f7b965c9073 in 588ms, sequenceid=121, compaction requested=false 2024-11-12T15:12:48,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for e4f31acd4066dc4cd42e4f7b965c9073: 2024-11-12T15:12:48,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. for snapshot_1731424367433_default_test-1731424350753 completed. 2024-11-12T15:12:48,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.' region-info for snapshot=snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:48,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:12:48,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824] hfiles 2024-11-12T15:12:48,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 for snapshot=snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:48,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae] hfiles 2024-11-12T15:12:48,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae for snapshot=snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:48,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb] hfiles 2024-11-12T15:12:48,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb for snapshot=snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:48,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741864_1040 (size=185) 2024-11-12T15:12:48,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:12:48,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-12T15:12:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-12T15:12:48,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424367433_default_test-1731424350753 on region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:48,806 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:12:48,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T15:12:48,812 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:12:48,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073 in 889 msec 2024-11-12T15:12:48,814 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:12:48,820 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-12T15:12:48,820 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-12T15:12:48,821 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:48,822 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-12T15:12:48,822 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:48,823 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-12T15:12:48,823 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:12:48,824 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-12T15:12:48,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741865_1041 (size=49) 2024-11-12T15:12:49,179 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-12T15:12:49,245 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:12:49,245 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:49,246 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:49,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741866_1042 (size=1146) 2024-11-12T15:12:49,404 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-12T15:12:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:49,665 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:12:49,678 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:12:49,679 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424367433_default_test-1731424350753 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:49,680 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:12:49,681 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-12T15:12:49,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1731424367433_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } in 2.2130 sec 2024-11-12T15:12:49,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-12T15:12:49,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:49,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-12T15:12:49,793 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:49,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1731424350753 2024-11-12T15:12:49,795 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1731424350753 Metrics about Tables on a single HBase RegionServer 2024-11-12T15:12:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T15:12:51,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1731424350753 completed 2024-11-12T15:12:51,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } 2024-11-12T15:12:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424371645 (current time:1731424371645). 2024-11-12T15:12:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:12:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424371644_ns2_test-17314243507531 VERSION not specified, setting to 2 2024-11-12T15:12:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dabf816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:12:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:12:51,647 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:12:51,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:12:51,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:12:51,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48dbbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:51,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:12:51,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:12:51,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:51,649 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:12:51,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f409e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:12:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:12:51,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:12:51,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:12:51,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:12:51,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:12:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:12:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:12:51,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:12:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:12:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } 2024-11-12T15:12:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-12T15:12:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T15:12:51,659 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:12:51,661 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:12:51,664 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:12:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741867_1043 (size=86) 2024-11-12T15:12:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T15:12:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T15:12:52,072 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:12:52,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe}] 2024-11-12T15:12:52,073 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:52,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-12T15:12:52,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:52,226 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 0e9acca221b224195ce9cae6845295fe 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-12T15:12:52,250 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/.tmp/f/9e5b564cc6c04f729750108536473385 is 37, key is row10/f:q1/1731424354289/Put/seqid=0 2024-11-12T15:12:52,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741868_1044 (size=8413) 2024-11-12T15:12:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T15:12:52,663 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/.tmp/f/9e5b564cc6c04f729750108536473385 2024-11-12T15:12:52,673 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/.tmp/f/9e5b564cc6c04f729750108536473385 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 2024-11-12T15:12:52,683 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385, entries=99, sequenceid=103, filesize=8.2 K 2024-11-12T15:12:52,684 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 0e9acca221b224195ce9cae6845295fe in 458ms, sequenceid=103, compaction requested=false 2024-11-12T15:12:52,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 0e9acca221b224195ce9cae6845295fe: 2024-11-12T15:12:52,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. for snapshot_1731424371644_ns2_test-17314243507531 completed. 2024-11-12T15:12:52,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.' region-info for snapshot=snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:52,685 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:12:52,685 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385] hfiles 2024-11-12T15:12:52,685 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 for snapshot=snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:52,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741869_1045 (size=91) 2024-11-12T15:12:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T15:12:53,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:12:53,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-12T15:12:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-12T15:12:53,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424371644_ns2_test-17314243507531 on region 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:53,112 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:12:53,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T15:12:53,116 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:12:53,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe in 1.0410 sec 2024-11-12T15:12:53,118 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:12:53,119 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:12:53,119 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:53,121 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:53,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741870_1046 (size=456) 2024-11-12T15:12:53,544 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:12:53,551 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:12:53,552 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424371644_ns2_test-17314243507531 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:53,554 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:12:53,554 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-12T15:12:53,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1731424371644_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } in 1.8970 sec 2024-11-12T15:12:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T15:12:53,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17314243507531 completed 2024-11-12T15:12:53,804 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1731424360169 2024-11-12T15:12:53,804 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-12T15:12:53,806 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-12T15:12:53,807 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424360169_test-1731424350753 2024-11-12T15:12:53,807 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424367433_default_test-1731424350753 to hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753/ 2024-11-12T15:12:53,812 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:12:53,844 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:53,844 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753/.hbase-snapshot/.tmp/snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:53,847 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:12:53,855 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424367433_default_test-1731424350753 to hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753/.hbase-snapshot/.tmp/snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:53,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741872_1048 (size=84) 2024-11-12T15:12:53,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741871_1047 (size=1146) 2024-11-12T15:12:54,275 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:54,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:54,276 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,297 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-5764266168045239348.jar 2024-11-12T15:12:55,298 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,298 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:55,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:12:55,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:12:55,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:12:55,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:12:55,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:12:55,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:12:55,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:12:55,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:12:55,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:12:55,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:12:55,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:12:55,305 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:55,305 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:55,306 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:12:55,306 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:55,306 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:55,306 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:12:55,307 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:12:55,372 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:12:55,406 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424367433_default_test-1731424350753' hfile list 2024-11-12T15:12:55,413 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=18.8 K 2024-11-12T15:12:55,629 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local335276948_0001_f3e6d8db-62e3-46bf-b7a4-f7eb936737e0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:12:55,629 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local335276948_0001_f3e6d8db-62e3-46bf-b7a4-f7eb936737e0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:12:55,872 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T15:12:55,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741873_1049 (size=8413) 2024-11-12T15:12:55,965 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 output=hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 2024-11-12T15:12:55,967 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.096M/sec 2024-11-12T15:12:56,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741874_1050 (size=5484) 2024-11-12T15:12:56,028 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb output=hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb 2024-11-12T15:12:56,028 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.122M/sec 2024-11-12T15:12:56,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741875_1051 (size=5387) 2024-11-12T15:12:56,107 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae output=hdfs://localhost:33269/backupUT/backup_1731424360169/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae 2024-11-12T15:12:56,107 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5387 (5.3 K) time=0sec 0.078M/sec 2024-11-12T15:12:56,737 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:12:56,738 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:12:56,747 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:56,747 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424367433_default_test-1731424350753 finished. 2024-11-12T15:12:56,747 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424360169_ns2:test-17314243507531 2024-11-12T15:12:56,747 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424371644_ns2_test-17314243507531 to hdfs://localhost:33269/backupUT/backup_1731424360169/ns2/test-17314243507531/ 2024-11-12T15:12:56,747 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:12:56,778 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:12:56,778 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424360169/ns2/test-17314243507531, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424360169/ns2/test-17314243507531/.hbase-snapshot/.tmp/snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:56,780 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:12:56,786 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424371644_ns2_test-17314243507531 to hdfs://localhost:33269/backupUT/backup_1731424360169/ns2/test-17314243507531/.hbase-snapshot/.tmp/snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:56,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741877_1053 (size=86) 2024-11-12T15:12:56,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741876_1052 (size=456) 2024-11-12T15:12:57,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:57,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:57,202 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-2448416671292443384.jar 2024-11-12T15:12:58,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:12:58,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:12:58,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:12:58,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:12:58,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:12:58,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:12:58,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:12:58,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:12:58,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:12:58,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:12:58,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:12:58,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:12:58,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:58,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:58,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:12:58,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:58,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:12:58,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:12:58,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:12:58,198 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:12:58,224 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424371644_ns2_test-17314243507531' hfile list 2024-11-12T15:12:58,227 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=8.2 K 2024-11-12T15:12:58,306 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local514928555_0002_9f9e7d8c-beb0-436c-8e43-b91b89946e30/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:12:58,306 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local514928555_0002_9f9e7d8c-beb0-436c-8e43-b91b89946e30/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:12:58,399 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T15:12:58,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741878_1054 (size=8413) 2024-11-12T15:12:58,448 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 output=hdfs://localhost:33269/backupUT/backup_1731424360169/ns2/test-17314243507531/archive/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 2024-11-12T15:12:58,448 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.187M/sec 2024-11-12T15:12:59,366 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:12:59,368 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:12:59,378 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:59,378 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424371644_ns2_test-17314243507531 finished. 2024-11-12T15:12:59,378 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1731424350753 2024-11-12T15:12:59,379 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17314243507531 2024-11-12T15:12:59,426 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731424360169 2024-11-12T15:12:59,426 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-12T15:12:59,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741879_1055 (size=130) 2024-11-12T15:12:59,835 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33269/backupUT/backup_1731424360169/.backup.manifest 2024-11-12T15:12:59,838 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731424360169 finished: type=FULL,tablelist=test-1731424350753;ns2:test-17314243507531,targetRootDir=hdfs://localhost:33269/backupUT,startts=1731424366974,completets=1731424379424,bytescopied=0 2024-11-12T15:12:59,838 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-12T15:12:59,838 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:59,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424367433_default_test-1731424350753" type: DISABLED 2024-11-12T15:12:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424367433_default_test-1731424350753 2024-11-12T15:12:59,847 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424367433_default_test-1731424350753 for backup backup_1731424360169 succeeded. 2024-11-12T15:12:59,847 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:59,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424371644_ns2_test-17314243507531" type: DISABLED 2024-11-12T15:12:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424371644_ns2_test-17314243507531 2024-11-12T15:12:59,853 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424371644_ns2_test-17314243507531 for backup backup_1731424360169 succeeded. 2024-11-12T15:12:59,854 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T15:12:59,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T15:12:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T15:12:59,873 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T15:12:59,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T15:12:59,902 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731424360169 completed. 2024-11-12T15:12:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:12:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:12:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1731424360169', locateType=CURRENT is [region=backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:13:00,200 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1731424350753 2024-11-12T15:13:00,244 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1731424350753 2024-11-12T15:13:00,529 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1731424350753 to Mob enabled CF 2024-11-12T15:13:00,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17314243507531', row='row-t20', locateType=CURRENT is [region=ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:13:00,568 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17314243507531 2024-11-12T15:13:00,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.3 split test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:00,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] assignment.SplitTableRegionProcedure(210): test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. splitKey isn't explicitly specified, will try to find a best split key from RS 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:00,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 because midkey is the same as first or last row 2024-11-12T15:13:00,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae because midkey is the same as first or last row 2024-11-12T15:13:00,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb because midkey is the same as first or last row 2024-11-12T15:13:00,581 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HRegion(2902): Flushing e4f31acd4066dc4cd42e4f7b965c9073 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-12T15:13:00,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f/fec72425894d47d99a52cb4e4acf2791 is 59, key is row-test-1731424350753-1-10/f:q1/1731424379956/Put/seqid=0 2024-11-12T15:13:00,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741880_1056 (size=10796) 2024-11-12T15:13:01,017 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f/fec72425894d47d99a52cb4e4acf2791 2024-11-12T15:13:01,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 is 63, key is row-test-1731424350753-3-100/mob:q1/1731424380507/Put/seqid=0 2024-11-12T15:13:01,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741881_1057 (size=5591) 2024-11-12T15:13:01,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:01,467 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:01,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/852c4ba01de54a449cd4c2a8fa6b7896, store: [table=test-1731424350753 family=mob region=e4f31acd4066dc4cd42e4f7b965c9073] 2024-11-12T15:13:01,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/852c4ba01de54a449cd4c2a8fa6b7896 is 192, key is row-test-1731424350753-3-100/mob:q1/1731424380507/Put/seqid=0 2024-11-12T15:13:01,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741882_1058 (size=13760) 2024-11-12T15:13:01,875 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/852c4ba01de54a449cd4c2a8fa6b7896 2024-11-12T15:13:01,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f/fec72425894d47d99a52cb4e4acf2791 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791 2024-11-12T15:13:01,894 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791, entries=99, sequenceid=334, filesize=10.5 K 2024-11-12T15:13:01,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/mob/852c4ba01de54a449cd4c2a8fa6b7896 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896 2024-11-12T15:13:01,906 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896, entries=111, sequenceid=334, filesize=13.4 K 2024-11-12T15:13:01,910 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for e4f31acd4066dc4cd42e4f7b965c9073 in 1329ms, sequenceid=334, compaction requested=false 2024-11-12T15:13:01,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.HRegion(2603): Flush status journal for e4f31acd4066dc4cd42e4f7b965c9073: 2024-11-12T15:13:01,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791 because midkey is the same as first or last row 2024-11-12T15:13:01,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae because midkey is the same as first or last row 2024-11-12T15:13:01,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896 because midkey is the same as first or last row 2024-11-12T15:13:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T15:13:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 99 connection: 172.17.0.3:34912 deadline: 1731424440575, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-12T15:13:01,920 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:01,920 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1731424350753 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1168) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:01,921 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T15:13:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731424350753,, stopping at row=test-1731424350753 ,, for max=2147483647 with caching=100 2024-11-12T15:13:01,937 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731424350753,, stopping at row=test-1731424350753 ,, for max=2147483647 with caching=100 2024-11-12T15:13:01,950 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1382 2024-11-12T15:13:01,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:01,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,012 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T15:13:02,021 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731424381950 2024-11-12T15:13:02,030 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731424360169} 2024-11-12T15:13:02,030 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-12T15:13:02,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:02,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424382120 (current time:1731424382120). 2024-11-12T15:13:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T15:13:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553f434d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:02,125 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:02,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:02,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:02,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2561399c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:02,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:02,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:02,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:02,130 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:02,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4725af02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:02,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:02,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:02,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:02,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:02,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-12T15:13:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:02,145 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:02,148 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:02,153 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:02,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741883_1059 (size=52) 2024-11-12T15:13:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:02,566 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:02,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450}] 2024-11-12T15:13:02,567 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:02,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-12T15:13:02,721 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:02,721 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 9ce3fb3bd1b279fea0145fdedac19450 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-12T15:13:02,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/ae0c644b692448d39ced11a9e55b9981 is 158, key is trslm:hdfs://localhost:33269/backupUT\x00ns2:test-17314243507531/meta:log-roll-map/1731424379415/Put/seqid=0 2024-11-12T15:13:02,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741884_1060 (size=5914) 2024-11-12T15:13:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:03,144 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/ae0c644b692448d39ced11a9e55b9981 2024-11-12T15:13:03,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/74df3208bc1741599a024776009497ed is 433, key is session:backup_1731424360169/session:context/1731424379879/Put/seqid=0 2024-11-12T15:13:03,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741885_1061 (size=5535) 2024-11-12T15:13:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:03,567 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/74df3208bc1741599a024776009497ed 2024-11-12T15:13:03,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/ae0c644b692448d39ced11a9e55b9981 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 2024-11-12T15:13:03,588 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981, entries=6, sequenceid=17, filesize=5.8 K 2024-11-12T15:13:03,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/74df3208bc1741599a024776009497ed as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed 2024-11-12T15:13:03,595 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed, entries=2, sequenceid=17, filesize=5.4 K 2024-11-12T15:13:03,596 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 875ms, sequenceid=17, compaction requested=false 2024-11-12T15:13:03,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 9ce3fb3bd1b279fea0145fdedac19450: 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. for snapshot_backup_system completed. 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.' region-info for snapshot=snapshot_backup_system 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981] hfiles 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 for snapshot=snapshot_backup_system 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed] hfiles 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d for snapshot=snapshot_backup_system 2024-11-12T15:13:03,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed for snapshot=snapshot_backup_system 2024-11-12T15:13:03,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741886_1062 (size=173) 2024-11-12T15:13:04,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:04,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-12T15:13:04,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-12T15:13:04,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:04,004 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:04,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-12T15:13:04,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 in 1.4390 sec 2024-11-12T15:13:04,007 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:04,008 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:04,009 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:04,009 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T15:13:04,010 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T15:13:04,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741887_1063 (size=810) 2024-11-12T15:13:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:04,427 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:04,436 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:04,437 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_backup_system 2024-11-12T15:13:04,438 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:04,438 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-12T15:13:04,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2970 sec 2024-11-12T15:13:06,200 DEBUG [master/2b6d221c5cde:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731424350753 2024-11-12T15:13:06,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741888_1064 (size=950) 2024-11-12T15:13:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-12T15:13:06,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T15:13:06,314 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731424381950 started at 1731424386314. 2024-11-12T15:13:06,318 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731424381950 has been started. 2024-11-12T15:13:06,325 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [test-1731424350753, ns2:test-17314243507531] 2024-11-12T15:13:06,331 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1731424351655for backupID backup_1731424381950 2024-11-12T15:13:06,331 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-12T15:13:06,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-12T15:13:06,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T15:13:06,332 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T15:13:06,332 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T15:13:06,332 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T15:13:06,333 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T15:13:06,335 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,335 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:06,582 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:13:06,582 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:13:06,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:13:06,582 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,582 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T15:13:06,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:06,583 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T15:13:06,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:06,584 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T15:13:06,584 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T15:13:06,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T15:13:06,584 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T15:13:06,584 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T15:13:06,584 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T15:13:06,584 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T15:13:06,584 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T15:13:06,618 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000002.908 2024-11-12T15:13:06,618 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000003.950 2024-11-12T15:13:06,619 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:06,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:06,620 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,620 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T15:13:06,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:06,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:06,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:06,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731424350753 in 421 msec 2024-11-12T15:13:06,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:06,623 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T15:13:06,623 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4270a503[Count = 0] remaining members to acquire global barrier 2024-11-12T15:13:06,623 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T15:13:06,623 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,724 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,724 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T15:13:06,724 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,724 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T15:13:06,724 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T15:13:06,724 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,724 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731424367172 highest: 1731424367172 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,725 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1:(num 1731424367105) roll requested 2024-11-12T15:13:06,743 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725, exclude list is [], retry=0 2024-11-12T15:13:06,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:06,748 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 2024-11-12T15:13:06,749 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:06,749 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 is not closed yet, will try archiving it next time 2024-11-12T15:13:06,749 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.meta:.meta(num 1731424367144) roll requested 2024-11-12T15:13:06,749 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424351655 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424351655 2024-11-12T15:13:06,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741857_1033 (size=30490) 2024-11-12T15:13:06,750 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 not finished, retry = 0 2024-11-12T15:13:06,765 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta, exclude list is [], retry=0 2024-11-12T15:13:06,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:06,770 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta 2024-11-12T15:13:06,770 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:06,770 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta is not closed yet, will try archiving it next time 2024-11-12T15:13:06,770 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0:(num 1731424367172) roll requested 2024-11-12T15:13:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741858_1034 (size=93) 2024-11-12T15:13:06,787 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta 2024-11-12T15:13:06,790 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771, exclude list is [], retry=0 2024-11-12T15:13:06,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:06,799 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 2024-11-12T15:13:06,799 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:06,799 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 is not closed yet, will try archiving it next time 2024-11-12T15:13:06,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741859_1035 (size=1198) 2024-11-12T15:13:06,806 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 82 2024-11-12T15:13:06,806 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731424386771 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:06,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:06,850 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T15:13:06,850 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '2b6d221c5cde,34071,1731424341329' in zk 2024-11-12T15:13:06,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,871 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T15:13:06,872 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:13:06,872 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T15:13:06,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:06,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:06,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:06,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:06,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:06,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:06,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:06,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '2b6d221c5cde,34071,1731424341329': 2024-11-12T15:13:06,879 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '2b6d221c5cde,34071,1731424341329' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T15:13:06,880 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T15:13:06,880 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T15:13:06,880 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T15:13:06,880 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T15:13:06,880 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T15:13:06,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:06,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:06,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:13:06,905 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:06,910 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,912 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:13:06,913 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:13:06,914 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:06,915 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,917 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:06,920 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:06,921 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:06,924 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,924 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:06,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:13:06,935 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:06,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:06,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:06,936 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:13:06,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:13:06,937 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:13:06,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:13:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T15:13:06,947 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {2b6d221c5cde:34071=1731424351655} newestTimestamps: {2b6d221c5cde:34071=1731424367172} 2024-11-12T15:13:06,949 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta 2024-11-12T15:13:06,949 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta 2024-11-12T15:13:06,949 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 2024-11-12T15:13:06,949 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 2024-11-12T15:13:06,950 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 2024-11-12T15:13:06,950 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 2024-11-12T15:13:06,950 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta 2024-11-12T15:13:06,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741892_1068 (size=950) 2024-11-12T15:13:06,966 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1731424350753 target: hdfs://localhost:33269/backupUT/backup_1731424381950/default/test-1731424350753 descriptor: 'test-1731424350753', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:06,966 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:06,970 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1731424350753 2024-11-12T15:13:06,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741893_1069 (size=53) 2024-11-12T15:13:07,391 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1731424350753 2024-11-12T15:13:07,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741894_1070 (size=364) 2024-11-12T15:13:07,813 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17314243507531 target: hdfs://localhost:33269/backupUT/backup_1731424381950/ns2/test-17314243507531 descriptor: 'ns2:test-17314243507531', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:07,813 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:07,815 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17314243507531 2024-11-12T15:13:07,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741895_1071 (size=50) 2024-11-12T15:13:08,226 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17314243507531 2024-11-12T15:13:08,237 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:33269/backupUT/.tmp/backup_1731424381950 from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172;hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 2024-11-12T15:13:08,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530b2482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:08,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:08,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:08,240 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:08,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:08,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:08,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a40a36e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:08,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:08,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:08,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:08,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:08,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1552e3b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:08,244 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T15:13:08,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,43085,1731424339322 2024-11-12T15:13:08,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23597d9 2024-11-12T15:13:08,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T15:13:08,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T15:13:08,255 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-12T15:13:08,255 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1731424350753 2024-11-12T15:13:08,255 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731424350753,, stopping at row=test-1731424350753 ,, for max=2147483647 with caching=100 2024-11-12T15:13:08,255 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:08,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:08,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:08,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:08,261 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1731424350753: test-1731424350753; 2024-11-12T15:13:08,261 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17314243507531 2024-11-12T15:13:08,262 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17314243507531,, stopping at row=ns2:test-17314243507531 ,, for max=2147483647 with caching=100 2024-11-12T15:13:08,264 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17314243507531: ns2:test-17314243507531; 2024-11-12T15:13:08,264 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-12T15:13:08,265 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/hbase-staging/partitions_532a7dc7-f4e1-460c-90e6-ec7b1ce4b19b 2024-11-12T15:13:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741896_1072 (size=192) 2024-11-12T15:13:08,291 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:08,292 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:08,293 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,318 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-8456992739967893929.jar 2024-11-12T15:13:09,319 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,319 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,320 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,320 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,320 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,321 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,321 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,322 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,322 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,322 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:09,323 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:09,323 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:09,323 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:09,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:09,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:09,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:09,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:09,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:09,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:09,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:09,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:09,331 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:09,331 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: default:test-1731424350753,ns2:test-17314243507531 2024-11-12T15:13:09,331 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:13:09,331 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:13:09,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:09,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:09,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:09,332 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:10,297 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-6344493209059805587.jar 2024-11-12T15:13:10,307 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:10,331 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 for WAL files 2024-11-12T15:13:10,333 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 2024-11-12T15:13:10,334 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 for WAL files 2024-11-12T15:13:10,335 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 2024-11-12T15:13:10,445 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local1704506544_0003_f4249484-d9c3-49cd-8a42-69a9c777c746/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:13:10,445 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local1704506544_0003_f4249484-d9c3-49cd-8a42-69a9c777c746/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:13:10,527 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 for hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 (-9223372036854775808:9223372036854775807) length:30490 2024-11-12T15:13:10,547 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-12T15:13:10,547 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-12T15:13:10,585 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 for hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 (-9223372036854775808:9223372036854775807) length:1198 2024-11-12T15:13:10,588 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-12T15:13:10,589 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-12T15:13:10,664 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136b8009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:10,665 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:10,665 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:10,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:10,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:10,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:10,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a82215e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:10,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:10,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:10,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:10,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:10,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b0b053, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:10,669 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:10,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:10,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:10,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:10,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17314243507531', row='row-t20', locateType=CURRENT is [region=ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:13:10,673 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-12T15:13:10,674 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:13:10,674 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:10,674 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:10,674 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:10,674 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-12T15:13:10,674 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 2b6d221c5cde 2024-11-12T15:13:10,678 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/backupUT/.tmp/backup_1731424381950/_temporary/0/_temporary/attempt_local1704506544_0003_r_000000_0/ns2/test-17314243507531/f/e85d40a893e34fbabeea434beec1cd2a is 38, key is row-t20/f:q1/1731424380553/Put/seqid=0 2024-11-12T15:13:10,681 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [2b6d221c5cde:34071] Specified favored nodes: [2b6d221c5cde:34071] 2024-11-12T15:13:10,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741897_1073 (size=5275) 2024-11-12T15:13:11,117 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5612d876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:11,117 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:11,117 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:11,118 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:11,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:11,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:11,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716971b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:11,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:11,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:11,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:11,120 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:11,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c81b0b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:11,121 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:11,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:11,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:11,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:11,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731424350753', row='row-test-1731424350753-1-0', locateType=CURRENT is [region=test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., hostname=2b6d221c5cde,34071,1731424341329, seqNum=107] 2024-11-12T15:13:11,125 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-12T15:13:11,125 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:13:11,125 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:11,125 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:11,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:11,125 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731424350753-1-0] 2024-11-12T15:13:11,125 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 2b6d221c5cde 2024-11-12T15:13:11,139 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773d19ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:11,139 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:11,139 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:11,140 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:11,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:11,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:11,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d8a389, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:11,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:11,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:11,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:11,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:11,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71be520b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:11,143 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:11,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:11,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:11,145 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:11,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1731424350753', row='row-test-1731424350753-3-0', locateType=CURRENT is [region=test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., hostname=2b6d221c5cde,34071,1731424341329, seqNum=107] 2024-11-12T15:13:11,147 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-12T15:13:11,147 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:13:11,147 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:11,147 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:11,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:11,147 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1731424350753-3-0] 2024-11-12T15:13:11,147 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 2b6d221c5cde 2024-11-12T15:13:11,158 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/backupUT/.tmp/backup_1731424381950/_temporary/0/_temporary/attempt_local1704506544_0003_r_000001_0/default/test-1731424350753/f/ca5fd046e7b449e38121d77cdccf4ec4 is 59, key is row-test-1731424350753-1-10/f:q1/1731424379956/Put/seqid=0 2024-11-12T15:13:11,160 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [2b6d221c5cde:34071] Specified favored nodes: [2b6d221c5cde:34071] 2024-11-12T15:13:11,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741898_1074 (size=10770) 2024-11-12T15:13:11,564 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/backupUT/.tmp/backup_1731424381950/_temporary/0/_temporary/attempt_local1704506544_0003_r_000001_0/default/test-1731424350753/mob/77006720be494a638fc06a17e410a74c is 63, key is row-test-1731424350753-3-100/mob:q1/1731424380507/Put/seqid=0 2024-11-12T15:13:11,567 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [2b6d221c5cde:34071] Specified favored nodes: [2b6d221c5cde:34071] 2024-11-12T15:13:11,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741899_1075 (size=11811) 2024-11-12T15:13:12,490 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:33269/backupUT 2024-11-12T15:13:12,491 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1731424381950 2024-11-12T15:13:12,491 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-12T15:13:12,723 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:33269/backupUT/.tmp/backup_1731424381950, hdfs://localhost:33269/backupUT] 2024-11-12T15:13:12,788 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:12,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741900_1076 (size=5275) 2024-11-12T15:13:13,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741901_1077 (size=11811) 2024-11-12T15:13:13,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741902_1078 (size=10770) 2024-11-12T15:13:14,394 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-12T15:13:14,402 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1731424381950 2024-11-12T15:13:14,402 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-12T15:13:14,403 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1426266852_0004 completed: true true 2024-11-12T15:13:14,404 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3823567 FILE: Number of bytes written=6205860 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231467 HDFS: Number of bytes written=2433185 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=251 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-12T15:13:14,404 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:33269/backupUT/.tmp/backup_1731424381950 to hdfs://localhost:33269/backupUT finished. 2024-11-12T15:13:14,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:test-1731424350753:', locateType=CURRENT is [region=backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61., hostname=2b6d221c5cde,34071,1731424341329, seqNum=2] 2024-11-12T15:13:14,419 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731424381950 2024-11-12T15:13:14,422 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731424360169} 2024-11-12T15:13:14,422 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-12T15:13:14,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741903_1079 (size=384) 2024-11-12T15:13:14,829 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33269/backupUT/backup_1731424381950/.backup.manifest 2024-11-12T15:13:14,829 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731424381950 finished: type=INCREMENTAL,tablelist=test-1731424350753;ns2:test-17314243507531,targetRootDir=hdfs://localhost:33269/backupUT,startts=1731424386314,completets=1731424394419,bytescopied=0 2024-11-12T15:13:14,829 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T15:13:14,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T15:13:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T15:13:14,840 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T15:13:14,843 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T15:13:14,847 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731424381950 completed. 2024-11-12T15:13:14,847 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-12T15:13:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:14,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:14,865 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33269/backupUT/backup_1731424381950/.backup.manifest 2024-11-12T15:13:14,868 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1731424381950/.backup.manifest 2024-11-12T15:13:14,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1731424350753 from 'test-1731424350753', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1731424350753', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1731424350753 2024-11-12T15:13:14,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741904_1080 (size=908) 2024-11-12T15:13:15,284 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000003.950 2024-11-12T15:13:15,285 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000004.908 2024-11-12T15:13:15,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1731424350753}] 2024-11-12T15:13:15,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE}] 2024-11-12T15:13:15,296 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE 2024-11-12T15:13:15,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=CLOSING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:15,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE because future has completed 2024-11-12T15:13:15,301 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T15:13:15,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:13:15,455 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,455 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T15:13:15,455 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing e4f31acd4066dc4cd42e4f7b965c9073, disabling compactions & flushes 2024-11-12T15:13:15,455 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,455 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,455 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. after waiting 0 ms 2024-11-12T15:13:15,455 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,466 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-12T15:13:15,467 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:13:15,467 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,467 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for e4f31acd4066dc4cd42e4f7b965c9073: Waiting for close lock at 1731424395455Running coprocessor pre-close hooks at 1731424395455Disabling compacts and flushes for region at 1731424395455Disabling writes for close at 1731424395455Writing region close event to WAL at 1731424395458 (+3 ms)Running coprocessor post-close hooks at 1731424395467 (+9 ms)Closed at 1731424395467 2024-11-12T15:13:15,468 WARN [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: e4f31acd4066dc4cd42e4f7b965c9073 to self. 2024-11-12T15:13:15,470 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=CLOSED 2024-11-12T15:13:15,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:13:15,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T15:13:15,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 in 173 msec 2024-11-12T15:13:15,477 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE; state=CLOSED, location=2b6d221c5cde,34071,1731424341329; forceNewPlan=false, retain=true 2024-11-12T15:13:15,628 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:15,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE because future has completed 2024-11-12T15:13:15,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:13:15,793 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,793 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => e4f31acd4066dc4cd42e4f7b965c9073, NAME => 'test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:13:15,794 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:13:15,794 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1731424350753 e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,794 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:13:15,794 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,794 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,795 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,796 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName f 2024-11-12T15:13:15,796 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:15,803 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 2024-11-12T15:13:15,808 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791 2024-11-12T15:13:15,808 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:13:15,808 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,810 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName f2 2024-11-12T15:13:15,810 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:15,810 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:13:15,810 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,811 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4f31acd4066dc4cd42e4f7b965c9073 columnFamilyName mob 2024-11-12T15:13:15,812 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:15,820 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896 2024-11-12T15:13:15,826 DEBUG [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb 2024-11-12T15:13:15,826 INFO [StoreOpener-e4f31acd4066dc4cd42e4f7b965c9073-1 {}] regionserver.HStore(327): Store=e4f31acd4066dc4cd42e4f7b965c9073/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:13:15,827 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,828 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,829 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,830 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,831 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,831 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1731424350753 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T15:13:15,833 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,834 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened e4f31acd4066dc4cd42e4f7b965c9073; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10676774720, jitterRate=-0.005647867918014526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T15:13:15,834 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:15,838 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for e4f31acd4066dc4cd42e4f7b965c9073: Running coprocessor pre-open hook at 1731424395794Writing region info on filesystem at 1731424395794Initializing all the Stores at 1731424395795 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424395795Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424395795Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424395795Cleaning up temporary data from old regions at 1731424395831 (+36 ms)Running coprocessor post-open hooks at 1731424395834 (+3 ms)Region opened successfully at 1731424395838 (+4 ms) 2024-11-12T15:13:15,839 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., pid=46, masterSystemTime=1731424395789 2024-11-12T15:13:15,841 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,841 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:15,842 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=e4f31acd4066dc4cd42e4f7b965c9073, regionState=OPEN, openSeqNum=338, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:15,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:13:15,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-12T15:13:15,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073, server=2b6d221c5cde,34071,1731424341329 in 212 msec 2024-11-12T15:13:15,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-12T15:13:15,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1731424350753, region=e4f31acd4066dc4cd42e4f7b965c9073, REOPEN/MOVE in 552 msec 2024-11-12T15:13:15,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T15:13:15,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1731424350753 in 560 msec 2024-11-12T15:13:15,853 DEBUG [PEWorker-4 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1731424350753 2024-11-12T15:13:15,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f3/cca7d187282e4238a0f131484a6889ae 2024-11-12T15:13:15,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1731424350753 in 990 msec 2024-11-12T15:13:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T15:13:15,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1731424350753 completed 2024-11-12T15:13:15,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,906 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T15:13:15,910 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731424395864 2024-11-12T15:13:15,912 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1731424381950} 2024-11-12T15:13:15,912 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731424360169} 2024-11-12T15:13:15,912 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-12T15:13:15,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,957 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1731424350753 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1731424350753 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T15:13:15,960 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1731424395864,startts=0,failedts=1731424395958,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1731424350753 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-12T15:13:15,960 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-12T15:13:15,962 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-12T15:13:15,962 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T15:13:15,964 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-12T15:13:15,964 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-12T15:13:15,966 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T15:13:15,969 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1731424395864 failed. 2024-11-12T15:13:15,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:15,987 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1731424350753, ns2:test-17314243507531, ns3:test-17314243507532, ns4:test-17314243507533] 2024-11-12T15:13:15,987 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T15:13:15,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424395991 (current time:1731424395991). 2024-11-12T15:13:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T15:13:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2f0a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:15,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:15,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:15,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:15,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64dc7b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:15,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:15,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:15,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:15,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:15,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8ae5f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:15,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:15,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:15,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:15,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:15,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-12T15:13:16,002 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:16,003 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:16,006 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:16,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741905_1081 (size=52) 2024-11-12T15:13:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:16,412 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:16,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450}] 2024-11-12T15:13:16,413 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:16,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-12T15:13:16,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:16,566 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 9ce3fb3bd1b279fea0145fdedac19450 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-12T15:13:16,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 is 158, key is trslm:hdfs://localhost:33269/backupUT\x00ns2:test-17314243507531/meta:log-roll-map/1731424394409/Put/seqid=0 2024-11-12T15:13:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741906_1082 (size=5745) 2024-11-12T15:13:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:16,976 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 2024-11-12T15:13:16,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/ece27341f33b4851ad44a3c1b49adb51 is 1245, key is session:backup_1731424395864/session:context/1731424395965/Put/seqid=0 2024-11-12T15:13:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741907_1083 (size=6682) 2024-11-12T15:13:17,012 DEBUG [master/2b6d221c5cde:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731424350753 2024-11-12T15:13:17,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741908_1084 (size=950) 2024-11-12T15:13:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:17,392 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/ece27341f33b4851ad44a3c1b49adb51 2024-11-12T15:13:17,399 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 2024-11-12T15:13:17,405 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0, entries=4, sequenceid=31, filesize=5.6 K 2024-11-12T15:13:17,406 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/ece27341f33b4851ad44a3c1b49adb51 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 2024-11-12T15:13:17,413 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51, entries=3, sequenceid=31, filesize=6.5 K 2024-11-12T15:13:17,415 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 848ms, sequenceid=31, compaction requested=true 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 9ce3fb3bd1b279fea0145fdedac19450: 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. for snapshot_backup_system completed. 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.' region-info for snapshot=snapshot_backup_system 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0] hfiles 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 for snapshot=snapshot_backup_system 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 for snapshot=snapshot_backup_system 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51] hfiles 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d for snapshot=snapshot_backup_system 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed for snapshot=snapshot_backup_system 2024-11-12T15:13:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 for snapshot=snapshot_backup_system 2024-11-12T15:13:17,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741909_1085 (size=251) 2024-11-12T15:13:17,422 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000004.908 2024-11-12T15:13:17,422 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/.tabledesc/.tableinfo.0000000005.950 2024-11-12T15:13:17,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1731424350753 in 411 msec 2024-11-12T15:13:17,822 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:17,822 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-12T15:13:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-12T15:13:17,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:17,824 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:17,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-12T15:13:17,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 in 1.4130 sec 2024-11-12T15:13:17,827 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:17,828 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:17,829 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:17,829 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T15:13:17,830 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T15:13:17,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741910_1086 (size=888) 2024-11-12T15:13:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:18,231 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T15:13:18,244 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:18,254 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0e9acca221b224195ce9cae6845295fe, had cached 0 bytes from a total of 8413 2024-11-12T15:13:18,255 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:18,256 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_backup_system 2024-11-12T15:13:18,258 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:18,258 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-12T15:13:18,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2580 sec 2024-11-12T15:13:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-12T15:13:20,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T15:13:20,175 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731424395970 started at 1731424400175. 2024-11-12T15:13:20,179 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731424395970 has been started. 2024-11-12T15:13:20,185 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-12T15:13:20,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-12T15:13:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T15:13:20,186 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T15:13:20,186 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T15:13:20,186 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T15:13:20,186 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T15:13:20,187 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,187 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:20,250 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:13:20,251 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:13:20,251 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:13:20,252 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,252 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T15:13:20,253 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:20,254 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T15:13:20,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:20,256 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T15:13:20,256 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T15:13:20,257 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T15:13:20,257 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T15:13:20,257 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T15:13:20,257 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T15:13:20,257 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T15:13:20,257 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T15:13:20,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,270 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:20,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:20,271 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,271 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T15:13:20,271 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:20,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:20,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:20,272 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:20,273 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T15:13:20,273 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3b819bb[Count = 0] remaining members to acquire global barrier 2024-11-12T15:13:20,273 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T15:13:20,273 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,281 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,281 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,281 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T15:13:20,281 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,282 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T15:13:20,282 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T15:13:20,282 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,282 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731424386771 highest: 1731424386771 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,282 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1:(num 1731424386725) roll requested 2024-11-12T15:13:20,295 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424400282, exclude list is [], retry=0 2024-11-12T15:13:20,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:20,299 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424400282 2024-11-12T15:13:20,299 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:20,299 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 is not closed yet, will try archiving it next time 2024-11-12T15:13:20,299 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.meta:.meta(num 1731424386749) roll requested 2024-11-12T15:13:20,299 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424367105 2024-11-12T15:13:20,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741889_1065 (size=4927) 2024-11-12T15:13:20,300 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 not finished, retry = 0 2024-11-12T15:13:20,311 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta, exclude list is [], retry=0 2024-11-12T15:13:20,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:20,314 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta 2024-11-12T15:13:20,314 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:20,315 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta is not closed yet, will try archiving it next time 2024-11-12T15:13:20,315 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0:(num 1731424386771) roll requested 2024-11-12T15:13:20,316 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta not finished, retry = 0 2024-11-12T15:13:20,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741890_1066 (size=1903) 2024-11-12T15:13:20,326 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315, exclude list is [], retry=0 2024-11-12T15:13:20,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:20,330 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 2024-11-12T15:13:20,330 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:20,330 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 is not closed yet, will try archiving it next time 2024-11-12T15:13:20,331 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 not finished, retry = 0 2024-11-12T15:13:20,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741891_1067 (size=93) 2024-11-12T15:13:20,342 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-12T15:13:20,342 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731424400315 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:20,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:20,362 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T15:13:20,362 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '2b6d221c5cde,34071,1731424341329' in zk 2024-11-12T15:13:20,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,376 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T15:13:20,376 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:13:20,376 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T15:13:20,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:20,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:20,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:20,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:20,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:20,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:20,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:20,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '2b6d221c5cde,34071,1731424341329': 2024-11-12T15:13:20,382 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T15:13:20,382 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '2b6d221c5cde,34071,1731424341329' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T15:13:20,382 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T15:13:20,382 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T15:13:20,382 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T15:13:20,382 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T15:13:20,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:13:20,396 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:20,396 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:13:20,396 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:20,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:20,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:20,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,398 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:20,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:20,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:13:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:20,407 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:13:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:13:20,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:20,408 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:13:20,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:20,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T15:13:20,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T15:13:20,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:13:20,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-12T15:13:20,408 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:13:20,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:13:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T15:13:20,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-12T15:13:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424400417 (current time:1731424400417). 2024-11-12T15:13:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424400416_backup_system_bulk VERSION not specified, setting to 2 2024-11-12T15:13:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5acf91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:20,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c08931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:20,420 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:20,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbf1e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:20,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:20,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:20,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:20,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:20,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-12T15:13:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-12T15:13:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T15:13:20,427 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:20,428 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:20,430 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:20,432 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424386771 2024-11-12T15:13:20,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741914_1090 (size=76) 2024-11-12T15:13:20,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T15:13:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T15:13:20,838 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:20,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure fb8c7f767a06180d5cc67ec654094c61}] 2024-11-12T15:13:20,840 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:13:20,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-12T15:13:20,993 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:13:20,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for fb8c7f767a06180d5cc67ec654094c61: 2024-11-12T15:13:20,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. for snapshot_1731424400416_backup_system_bulk completed. 2024-11-12T15:13:20,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.' region-info for snapshot=snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:20,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:20,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T15:13:20,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T15:13:20,996 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-12T15:13:21,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741915_1091 (size=61) 2024-11-12T15:13:21,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T15:13:21,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:13:21,405 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T15:13:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T15:13:21,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424400416_backup_system_bulk on region fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:13:21,407 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:13:21,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T15:13:21,415 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:21,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure fb8c7f767a06180d5cc67ec654094c61 in 572 msec 2024-11-12T15:13:21,416 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:21,416 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:21,416 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:21,417 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:21,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741916_1092 (size=702) 2024-11-12T15:13:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T15:13:21,827 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:21,833 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:21,833 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424400416_backup_system_bulk to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:21,835 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:21,835 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-12T15:13:21,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1731424400416_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4100 sec 2024-11-12T15:13:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T15:13:22,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-12T15:13:22,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } 2024-11-12T15:13:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424402575 (current time:1731424402575). 2024-11-12T15:13:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424402574_ns3_test-17314243507532 VERSION not specified, setting to 2 2024-11-12T15:13:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4649ffe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:22,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:22,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:22,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:22,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587e0689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:22,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:22,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:22,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:22,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:22,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da5a0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:22,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:22,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:22,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:22,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:22,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } 2024-11-12T15:13:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-12T15:13:22,586 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T15:13:22,587 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:22,589 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:22,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741917_1093 (size=86) 2024-11-12T15:13:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T15:13:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T15:13:22,996 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:22,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure aa6d9848b308420055c29add59789661}] 2024-11-12T15:13:22,998 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure aa6d9848b308420055c29add59789661 2024-11-12T15:13:23,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-12T15:13:23,150 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:13:23,150 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for aa6d9848b308420055c29add59789661: 2024-11-12T15:13:23,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. for snapshot_1731424402574_ns3_test-17314243507532 completed. 2024-11-12T15:13:23,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661.' region-info for snapshot=snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:23,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:23,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T15:13:23,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741918_1094 (size=52) 2024-11-12T15:13:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T15:13:23,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:13:23,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T15:13:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T15:13:23,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424402574_ns3_test-17314243507532 on region aa6d9848b308420055c29add59789661 2024-11-12T15:13:23,563 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure aa6d9848b308420055c29add59789661 2024-11-12T15:13:23,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T15:13:23,567 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:23,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure aa6d9848b308420055c29add59789661 in 568 msec 2024-11-12T15:13:23,568 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:23,569 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:23,569 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:23,570 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:23,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741919_1095 (size=443) 2024-11-12T15:13:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T15:13:23,993 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:24,000 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:24,001 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424402574_ns3_test-17314243507532 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:24,002 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:24,002 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-12T15:13:24,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1731424402574_ns3_test-17314243507532 table=ns3:test-17314243507532 type=FLUSH ttl=0 } in 1.4180 sec 2024-11-12T15:13:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T15:13:24,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17314243507532 completed 2024-11-12T15:13:24,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } 2024-11-12T15:13:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424404734 (current time:1731424404734). 2024-11-12T15:13:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424404734_default_test-1731424350753 VERSION not specified, setting to 2 2024-11-12T15:13:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318b91f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:24,736 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:24,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:24,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:24,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e6983d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:24,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:24,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:24,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:24,739 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:24,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4987d42b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:24,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:24,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:24,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:24,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:24,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } 2024-11-12T15:13:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-12T15:13:24,748 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T15:13:24,749 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:24,752 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:24,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741920_1096 (size=84) 2024-11-12T15:13:24,764 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:24,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073}] 2024-11-12T15:13:24,765 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T15:13:24,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e4f31acd4066dc4cd42e4f7b965c9073: 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. for snapshot_1731424404734_default_test-1731424350753 completed. 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073.' region-info for snapshot=snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791] hfiles 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 for snapshot=snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791 for snapshot=snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896] hfiles 2024-11-12T15:13:24,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb for snapshot=snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:24,919 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896 for snapshot=snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:24,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741921_1097 (size=224) 2024-11-12T15:13:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T15:13:25,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:13:25,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-12T15:13:25,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-12T15:13:25,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424404734_default_test-1731424350753 on region e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:25,326 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:25,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T15:13:25,330 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:25,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e4f31acd4066dc4cd42e4f7b965c9073 in 563 msec 2024-11-12T15:13:25,331 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:25,332 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-12T15:13:25,333 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-12T15:13:25,333 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:25,333 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-12T15:13:25,333 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:25,334 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-12T15:13:25,334 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:13:25,335 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073] hfiles 2024-11-12T15:13:25,335 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:25,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741922_1098 (size=168) 2024-11-12T15:13:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T15:13:25,755 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:25,755 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:25,756 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:25,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741923_1099 (size=1347) 2024-11-12T15:13:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T15:13:26,180 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:26,189 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:26,189 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424404734_default_test-1731424350753 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:26,191 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:26,191 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-12T15:13:26,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1731424404734_default_test-1731424350753 table=test-1731424350753 type=FLUSH ttl=0 } in 1.4460 sec 2024-11-12T15:13:26,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9ce3fb3bd1b279fea0145fdedac19450, had cached 0 bytes from a total of 28925 2024-11-12T15:13:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T15:13:26,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1731424350753 completed 2024-11-12T15:13:26,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } 2024-11-12T15:13:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424406895 (current time:1731424406895). 2024-11-12T15:13:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424406894_ns2_test-17314243507531 VERSION not specified, setting to 2 2024-11-12T15:13:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515b8deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:26,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:26,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:26,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:26,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a873183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:26,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:26,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:26,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:26,900 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:26,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37aa0ace, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:26,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:26,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:26,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:26,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:26,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } 2024-11-12T15:13:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-12T15:13:26,913 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T15:13:26,914 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:26,927 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:26,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741924_1100 (size=86) 2024-11-12T15:13:26,942 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:26,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe}] 2024-11-12T15:13:26,943 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:13:26,980 INFO [master/2b6d221c5cde:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T15:13:26,980 INFO [master/2b6d221c5cde:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T15:13:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T15:13:27,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-12T15:13:27,096 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:13:27,096 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 0e9acca221b224195ce9cae6845295fe 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-12T15:13:27,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/.tmp/f/a6bd7113207c4e479116093e9d65c2a5 is 38, key is row-t20/f:q1/1731424380553/Put/seqid=0 2024-11-12T15:13:27,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741925_1101 (size=5134) 2024-11-12T15:13:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T15:13:27,513 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/.tmp/f/a6bd7113207c4e479116093e9d65c2a5 2024-11-12T15:13:27,518 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/.tmp/f/a6bd7113207c4e479116093e9d65c2a5 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/a6bd7113207c4e479116093e9d65c2a5 2024-11-12T15:13:27,523 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/a6bd7113207c4e479116093e9d65c2a5, entries=5, sequenceid=111, filesize=5.0 K 2024-11-12T15:13:27,524 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 0e9acca221b224195ce9cae6845295fe in 428ms, sequenceid=111, compaction requested=false 2024-11-12T15:13:27,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 0e9acca221b224195ce9cae6845295fe: 2024-11-12T15:13:27,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. for snapshot_1731424406894_ns2_test-17314243507531 completed. 2024-11-12T15:13:27,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe.' region-info for snapshot=snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:27,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:27,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/a6bd7113207c4e479116093e9d65c2a5] hfiles 2024-11-12T15:13:27,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 for snapshot=snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:27,525 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/a6bd7113207c4e479116093e9d65c2a5 for snapshot=snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741926_1102 (size=130) 2024-11-12T15:13:27,532 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:13:27,532 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-12T15:13:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-12T15:13:27,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424406894_ns2_test-17314243507531 on region 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:13:27,533 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:13:27,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T15:13:27,540 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:27,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 0e9acca221b224195ce9cae6845295fe in 595 msec 2024-11-12T15:13:27,541 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:27,542 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:27,542 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:27,543 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T15:13:27,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741927_1103 (size=496) 2024-11-12T15:13:27,960 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:27,970 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:27,971 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424406894_ns2_test-17314243507531 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:27,974 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:27,974 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-12T15:13:27,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1731424406894_ns2_test-17314243507531 table=ns2:test-17314243507531 type=FLUSH ttl=0 } in 1.0640 sec 2024-11-12T15:13:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T15:13:28,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17314243507531 completed 2024-11-12T15:13:28,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } 2024-11-12T15:13:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424408055 (current time:1731424408055). 2024-11-12T15:13:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1731424408054_ns4_test-17314243507533 VERSION not specified, setting to 2 2024-11-12T15:13:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab193d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:28,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:28,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:28,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:28,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d031ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:28,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:28,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:28,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:28,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:28,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddecddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:28,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:28,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:28,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:28,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:28,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } 2024-11-12T15:13:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-12T15:13:28,063 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T15:13:28,064 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:28,067 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741928_1104 (size=86) 2024-11-12T15:13:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T15:13:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T15:13:28,477 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:28,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b64e86ccc223899acbc0b981c82d38b4}] 2024-11-12T15:13:28,480 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:13:28,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-12T15:13:28,633 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:13:28,633 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for b64e86ccc223899acbc0b981c82d38b4: 2024-11-12T15:13:28,633 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. for snapshot_1731424408054_ns4_test-17314243507533 completed. 2024-11-12T15:13:28,633 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4.' region-info for snapshot=snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:28,634 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:28,634 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-12T15:13:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741929_1105 (size=52) 2024-11-12T15:13:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T15:13:29,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:13:29,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-12T15:13:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-12T15:13:29,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1731424408054_ns4_test-17314243507533 on region b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:13:29,048 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:13:29,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T15:13:29,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure b64e86ccc223899acbc0b981c82d38b4 in 572 msec 2024-11-12T15:13:29,054 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:29,056 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:29,057 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:29,057 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:29,059 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:29,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741930_1106 (size=443) 2024-11-12T15:13:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T15:13:29,478 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:29,484 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:29,485 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_1731424408054_ns4_test-17314243507533 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:29,486 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:29,486 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-12T15:13:29,487 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1731424408054_ns4_test-17314243507533 table=ns4:test-17314243507533 type=FLUSH ttl=0 } in 1.4240 sec 2024-11-12T15:13:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T15:13:30,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17314243507533 completed 2024-11-12T15:13:30,224 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1731424395970 2024-11-12T15:13:30,224 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-12T15:13:30,224 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-12T15:13:30,224 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424395970_backup:system_bulk 2024-11-12T15:13:30,225 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424400416_backup_system_bulk to hdfs://localhost:33269/backupUT/backup_1731424395970/backup/system_bulk/ 2024-11-12T15:13:30,225 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:13:30,254 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:13:30,254 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424395970/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424395970/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:30,256 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:13:30,260 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424400416_backup_system_bulk to hdfs://localhost:33269/backupUT/backup_1731424395970/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741931_1107 (size=702) 2024-11-12T15:13:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741932_1108 (size=76) 2024-11-12T15:13:30,672 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:30,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:30,673 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-2978545325408206699.jar 2024-11-12T15:13:31,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,606 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,607 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,608 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,608 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:31,608 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:31,608 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:31,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:31,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:31,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:31,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:31,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:31,610 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:31,610 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:31,610 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:31,610 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:31,611 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:31,611 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:31,611 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:31,611 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:31,611 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:31,612 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:31,612 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:31,648 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:31,675 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424400416_backup_system_bulk' hfile list 2024-11-12T15:13:31,755 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local1800388933_0005_88d59175-2452-41e2-a4e0-e9eb817f658f/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:13:31,755 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local1800388933_0005_88d59175-2452-41e2-a4e0-e9eb817f658f/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:13:32,793 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:13:32,794 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:13:32,800 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:32,800 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424400416_backup_system_bulk finished. 2024-11-12T15:13:32,800 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424395970_ns3:test-17314243507532 2024-11-12T15:13:32,800 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424402574_ns3_test-17314243507532 to hdfs://localhost:33269/backupUT/backup_1731424395970/ns3/test-17314243507532/ 2024-11-12T15:13:32,800 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:13:32,824 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:13:32,824 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424395970/ns3/test-17314243507532, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424395970/ns3/test-17314243507532/.hbase-snapshot/.tmp/snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:32,826 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:13:32,829 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424402574_ns3_test-17314243507532 to hdfs://localhost:33269/backupUT/backup_1731424395970/ns3/test-17314243507532/.hbase-snapshot/.tmp/snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:32,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741933_1109 (size=86) 2024-11-12T15:13:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741934_1110 (size=443) 2024-11-12T15:13:33,244 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:33,245 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:33,245 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,124 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-1881174802209373811.jar 2024-11-12T15:13:34,124 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:34,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:34,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:34,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:34,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:34,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:34,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:34,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:34,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:34,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:34,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:34,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:34,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:34,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:34,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:34,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:34,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:34,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:34,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:34,161 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:34,185 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424402574_ns3_test-17314243507532' hfile list 2024-11-12T15:13:34,275 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local926177484_0006_881564e4-2772-4ef0-9b4d-e84fc26659ba/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:13:34,275 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local926177484_0006_881564e4-2772-4ef0-9b4d-e84fc26659ba/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:13:35,312 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:13:35,315 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:13:35,324 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:35,324 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424402574_ns3_test-17314243507532 finished. 2024-11-12T15:13:35,324 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424395970_test-1731424350753 2024-11-12T15:13:35,324 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424404734_default_test-1731424350753 to hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/ 2024-11-12T15:13:35,324 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:13:35,352 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:13:35,352 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/.hbase-snapshot/.tmp/snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:35,354 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:13:35,360 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424404734_default_test-1731424350753 to hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/.hbase-snapshot/.tmp/snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:35,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741935_1111 (size=84) 2024-11-12T15:13:35,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741936_1112 (size=1347) 2024-11-12T15:13:35,772 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:35,773 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:35,773 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,647 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-1877184597340035798.jar 2024-11-12T15:13:36,647 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,647 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,648 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:36,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:36,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:36,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:36,649 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:36,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:36,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:36,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:36,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:36,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:36,650 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:36,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:36,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:36,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:36,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:36,651 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:36,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:36,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:36,652 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:36,685 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:36,709 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424404734_default_test-1731424350753' hfile list 2024-11-12T15:13:36,711 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=43.0 K 2024-11-12T15:13:36,786 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local489038279_0007_d89791c5-bc65-4b97-807e-0e23a8b744d8/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:13:36,786 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local489038279_0007_d89791c5-bc65-4b97-807e-0e23a8b744d8/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:13:36,855 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T15:13:36,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741937_1113 (size=13760) 2024-11-12T15:13:37,301 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896 output=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/852c4ba01de54a449cd4c2a8fa6b7896 2024-11-12T15:13:37,301 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-12T15:13:37,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741938_1114 (size=10796) 2024-11-12T15:13:37,726 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791 output=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/fec72425894d47d99a52cb4e4acf2791 2024-11-12T15:13:37,726 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=10796 (10.5 K) time=0sec 0.025M/sec 2024-11-12T15:13:37,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741939_1115 (size=8413) 2024-11-12T15:13:38,218 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 output=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f/245e1c5baac741b5b8cbe8ed5b46b824 2024-11-12T15:13:38,218 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-11-12T15:13:38,236 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 to hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/archive/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073" truncated to max limit (512 characters) 2024-11-12T15:13:38,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741940_1116 (size=5591) 2024-11-12T15:13:38,652 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/mobdir/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 output=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/archive/data/default/test-1731424350753/4d9c2cd7701f26365648a795879e4f18/mob/d41d8cd98f00b204e9800998ecf8427e20241112a7545eb187274a16922b255f76035ead_e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:13:38,652 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-11-12T15:13:38,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741941_1117 (size=5484) 2024-11-12T15:13:38,700 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb output=hdfs://localhost:33269/backupUT/backup_1731424395970/default/test-1731424350753/archive/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/mob/dffaf2b995334a6dad0b06c90b71accb 2024-11-12T15:13:38,700 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5484 (5.4 K) time=0sec 0.125M/sec 2024-11-12T15:13:38,829 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:13:38,830 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:13:38,838 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:38,838 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424404734_default_test-1731424350753 finished. 2024-11-12T15:13:38,838 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424395970_ns2:test-17314243507531 2024-11-12T15:13:38,838 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424406894_ns2_test-17314243507531 to hdfs://localhost:33269/backupUT/backup_1731424395970/ns2/test-17314243507531/ 2024-11-12T15:13:38,838 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:13:38,872 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:13:38,872 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424395970/ns2/test-17314243507531, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424395970/ns2/test-17314243507531/.hbase-snapshot/.tmp/snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:38,874 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:13:38,879 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424406894_ns2_test-17314243507531 to hdfs://localhost:33269/backupUT/backup_1731424395970/ns2/test-17314243507531/.hbase-snapshot/.tmp/snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:38,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741943_1119 (size=86) 2024-11-12T15:13:38,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741942_1118 (size=496) 2024-11-12T15:13:39,294 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:39,295 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:39,296 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,298 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-13133898593742462119.jar 2024-11-12T15:13:40,298 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,298 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,299 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:40,300 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:40,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:40,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:40,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:40,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:40,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:40,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:40,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:40,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:40,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:40,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:40,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:40,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:40,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:40,304 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:40,304 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:40,304 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:40,304 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:40,347 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:40,382 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424406894_ns2_test-17314243507531' hfile list 2024-11-12T15:13:40,385 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(754): export split=0 size=13.2 K 2024-11-12T15:13:40,483 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local485593941_0008_94462685-6efa-4199-b0e4-bba3e4b38a4b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:13:40,483 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local485593941_0008_94462685-6efa-4199-b0e4-bba3e4b38a4b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:13:40,565 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(237): Using bufferSize=128 M 2024-11-12T15:13:40,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741944_1120 (size=8413) 2024-11-12T15:13:40,989 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 output=hdfs://localhost:33269/backupUT/backup_1731424395970/ns2/test-17314243507531/archive/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/9e5b564cc6c04f729750108536473385 2024-11-12T15:13:40,989 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-11-12T15:13:41,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741945_1121 (size=5134) 2024-11-12T15:13:41,032 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(338): copy completed for input=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/a6bd7113207c4e479116093e9d65c2a5 output=hdfs://localhost:33269/backupUT/backup_1731424395970/ns2/test-17314243507531/archive/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/f/a6bd7113207c4e479116093e9d65c2a5 2024-11-12T15:13:41,033 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): size=5134 (5.0 K) time=0sec 0.132M/sec 2024-11-12T15:13:41,528 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:13:41,529 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:13:41,537 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:41,537 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424406894_ns2_test-17314243507531 finished. 2024-11-12T15:13:41,537 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1731424395970_ns4:test-17314243507533 2024-11-12T15:13:41,537 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1731424408054_ns4_test-17314243507533 to hdfs://localhost:33269/backupUT/backup_1731424395970/ns4/test-17314243507533/ 2024-11-12T15:13:41,537 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-12T15:13:41,571 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1085): inputFs=hdfs://localhost:33269, inputRoot=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265 2024-11-12T15:13:41,571 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1086): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_608511698_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33269/backupUT/backup_1731424395970/ns4/test-17314243507533, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33269/backupUT/backup_1731424395970/ns4/test-17314243507533/.hbase-snapshot/.tmp/snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:41,573 INFO [Time-limited test {}] snapshot.ExportSnapshot(1095): Verify the source snapshot's expiration status and integrity. 2024-11-12T15:13:41,578 INFO [Time-limited test {}] snapshot.ExportSnapshot(1153): Copy Snapshot Manifest from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_1731424408054_ns4_test-17314243507533 to hdfs://localhost:33269/backupUT/backup_1731424395970/ns4/test-17314243507533/.hbase-snapshot/.tmp/snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:41,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741946_1122 (size=86) 2024-11-12T15:13:41,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741947_1123 (size=443) 2024-11-12T15:13:41,994 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:41,994 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:41,995 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-14513144717816862197.jar 2024-11-12T15:13:42,969 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,970 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,971 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:42,972 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:42,972 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:42,972 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:42,972 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:42,972 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:42,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:42,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:42,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:42,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:42,973 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:42,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:42,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:42,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:42,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:42,974 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:42,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-12T15:13:42,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:42,975 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:43,008 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:43,030 INFO [Time-limited test {}] snapshot.ExportSnapshot(661): Loading Snapshot 'snapshot_1731424408054_ns4_test-17314243507533' hfile list 2024-11-12T15:13:43,110 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local1497328284_0009_a412b6d7-2c2c-4d2b-bf4f-d59f2185de79/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-12T15:13:43,111 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/mapreduce.cluster.local.dir/job_local1497328284_0009_a412b6d7-2c2c-4d2b-bf4f-d59f2185de79/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-12T15:13:44,149 INFO [Time-limited test {}] snapshot.ExportSnapshot(1210): Finalize the Snapshot Export 2024-11-12T15:13:44,150 INFO [Time-limited test {}] snapshot.ExportSnapshot(1221): Verify the exported snapshot's expiration status and integrity. 2024-11-12T15:13:44,155 INFO [Time-limited test {}] snapshot.ExportSnapshot(1227): Export Completed: snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:44,155 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1731424408054_ns4_test-17314243507533 finished. 2024-11-12T15:13:44,155 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-12T15:13:44,155 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17314243507532 2024-11-12T15:13:44,155 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1731424350753 2024-11-12T15:13:44,155 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17314243507531 2024-11-12T15:13:44,155 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17314243507533 2024-11-12T15:13:44,165 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731424395970 2024-11-12T15:13:44,165 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-12T15:13:44,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741948_1124 (size=209) 2024-11-12T15:13:44,571 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33269/backupUT/backup_1731424395970/.backup.manifest 2024-11-12T15:13:44,571 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1731424395970 finished: type=FULL,tablelist=backup:system_bulk;ns3:test-17314243507532;test-1731424350753;ns2:test-17314243507531;ns4:test-17314243507533,targetRootDir=hdfs://localhost:33269/backupUT,startts=1731424400175,completets=1731424424165,bytescopied=0 2024-11-12T15:13:44,571 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-12T15:13:44,571 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:44,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424400416_backup_system_bulk" type: DISABLED 2024-11-12T15:13:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424400416_backup_system_bulk 2024-11-12T15:13:44,576 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424400416_backup_system_bulk for backup backup_1731424395970 succeeded. 2024-11-12T15:13:44,576 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:44,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424402574_ns3_test-17314243507532" type: DISABLED 2024-11-12T15:13:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424402574_ns3_test-17314243507532 2024-11-12T15:13:44,581 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424402574_ns3_test-17314243507532 for backup backup_1731424395970 succeeded. 2024-11-12T15:13:44,581 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:44,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424404734_default_test-1731424350753" type: DISABLED 2024-11-12T15:13:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424404734_default_test-1731424350753 2024-11-12T15:13:44,585 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424404734_default_test-1731424350753 for backup backup_1731424395970 succeeded. 2024-11-12T15:13:44,585 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:44,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424406894_ns2_test-17314243507531" type: DISABLED 2024-11-12T15:13:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424406894_ns2_test-17314243507531 2024-11-12T15:13:44,589 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424406894_ns2_test-17314243507531 for backup backup_1731424395970 succeeded. 2024-11-12T15:13:44,589 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:44,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1731424408054_ns4_test-17314243507533" type: DISABLED 2024-11-12T15:13:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1731424408054_ns4_test-17314243507533 2024-11-12T15:13:44,592 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1731424408054_ns4_test-17314243507533 for backup backup_1731424395970 succeeded. 2024-11-12T15:13:44,593 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T15:13:44,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T15:13:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T15:13:44,601 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T15:13:44,603 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T15:13:44,608 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1731424395970 completed. 2024-11-12T15:13:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:44,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:48,232 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T15:13:49,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,713 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-12T15:13:49,719 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1731424429639 2024-11-12T15:13:49,723 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1731424395970} 2024-11-12T15:13:49,723 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-12T15:13:49,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:49,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424429868 (current time:1731424429868). 2024-11-12T15:13:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-12T15:13:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517676b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:49,873 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:49,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:49,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:49,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8bbacd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:49,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:49,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:49,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:49,876 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:49,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce8dc2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:49,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:49,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:49,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:49,883 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:49,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-12T15:13:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T15:13:49,888 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:49,890 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:49,895 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:49,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741949_1125 (size=52) 2024-11-12T15:13:49,909 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-12T15:13:49,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450}] 2024-11-12T15:13:49,911 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T15:13:50,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34071 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-12T15:13:50,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:50,066 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9ce3fb3bd1b279fea0145fdedac19450 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-12T15:13:50,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/59310573775849a5a04697e1a3a246c0 is 158, key is trslm:hdfs://localhost:33269/backupUT\x00ns2:test-17314243507531/meta:log-roll-map/1731424424159/Put/seqid=0 2024-11-12T15:13:50,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741950_1126 (size=6646) 2024-11-12T15:13:50,104 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/59310573775849a5a04697e1a3a246c0 2024-11-12T15:13:50,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/14347bca6aca4942922ff4c9589be733 is 826, key is session:backup_1731424395970/session:context/1731424424602/Put/seqid=0 2024-11-12T15:13:50,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741951_1127 (size=5928) 2024-11-12T15:13:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T15:13:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T15:13:50,525 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/14347bca6aca4942922ff4c9589be733 2024-11-12T15:13:50,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/59310573775849a5a04697e1a3a246c0 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0 2024-11-12T15:13:50,541 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0, entries=12, sequenceid=42, filesize=6.5 K 2024-11-12T15:13:50,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/14347bca6aca4942922ff4c9589be733 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733 2024-11-12T15:13:50,550 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733, entries=2, sequenceid=42, filesize=5.8 K 2024-11-12T15:13:50,551 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 485ms, sequenceid=42, compaction requested=true 2024-11-12T15:13:50,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9ce3fb3bd1b279fea0145fdedac19450: 2024-11-12T15:13:50,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. for snapshot_backup_system completed. 2024-11-12T15:13:50,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.' region-info for snapshot=snapshot_backup_system 2024-11-12T15:13:50,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0] hfiles 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 for snapshot=snapshot_backup_system 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 for snapshot=snapshot_backup_system 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0 for snapshot=snapshot_backup_system 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733] hfiles 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d for snapshot=snapshot_backup_system 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed for snapshot=snapshot_backup_system 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 for snapshot=snapshot_backup_system 2024-11-12T15:13:50,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733 for snapshot=snapshot_backup_system 2024-11-12T15:13:50,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741952_1128 (size=330) 2024-11-12T15:13:50,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:50,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-12T15:13:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-12T15:13:50,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:50,961 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:50,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T15:13:50,966 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-12T15:13:50,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450 in 1.0530 sec 2024-11-12T15:13:50,967 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:13:50,968 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:13:50,968 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-12T15:13:50,969 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-12T15:13:51,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741953_1129 (size=967) 2024-11-12T15:13:51,017 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:13:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T15:13:51,035 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:13:51,036 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/snapshot_backup_system 2024-11-12T15:13:51,039 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:13:51,039 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-12T15:13:51,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1550 sec 2024-11-12T15:13:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T15:13:52,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T15:13:52,034 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1731424429639 started at 1731424432034. 2024-11-12T15:13:52,036 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1731424429639 has been started. 2024-11-12T15:13:52,041 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, test-1731424350753, ns2:test-17314243507531, ns3:test-17314243507532, ns4:test-17314243507533] 2024-11-12T15:13:52,045 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1731424386771for backupID backup_1731424429639 2024-11-12T15:13:52,045 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-12T15:13:52,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-11-12T15:13:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-12T15:13:52,046 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-12T15:13:52,046 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-12T15:13:52,046 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-12T15:13:52,046 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-12T15:13:52,047 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,047 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:52,238 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:13:52,239 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,239 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:13:52,239 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-12T15:13:52,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:13:52,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:52,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-12T15:13:52,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:52,241 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-12T15:13:52,241 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-12T15:13:52,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-12T15:13:52,241 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-12T15:13:52,241 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-12T15:13:52,242 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-12T15:13:52,242 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-12T15:13:52,242 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure (rolllog) in zk 2024-11-12T15:13:52,258 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:52,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:52,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:52,259 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:52,259 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:52,259 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,260 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:52,260 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '2b6d221c5cde,34071,1731424341329' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-12T15:13:52,260 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@29c4e539[Count = 0] remaining members to acquire global barrier 2024-11-12T15:13:52,260 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-12T15:13:52,260 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,268 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] zookeeper.ZKUtil(111): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,268 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,268 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-12T15:13:52,268 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-12T15:13:52,268 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,268 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-12T15:13:52,269 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-12T15:13:52,269 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,269 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1731424400315 highest: 1731424400315 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,269 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1:(num 1731424400282) roll requested 2024-11-12T15:13:52,288 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424432269, exclude list is [], retry=0 2024-11-12T15:13:52,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:52,298 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424400282 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424432269 2024-11-12T15:13:52,298 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:52,298 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424400282 is not closed yet, will try archiving it next time 2024-11-12T15:13:52,298 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.meta:.meta(num 1731424400299) roll requested 2024-11-12T15:13:52,298 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424386725 2024-11-12T15:13:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741911_1087 (size=4710) 2024-11-12T15:13:52,312 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424432298.meta, exclude list is [], retry=0 2024-11-12T15:13:52,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:52,316 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424432298.meta 2024-11-12T15:13:52,317 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:52,317 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta is not closed yet, will try archiving it next time 2024-11-12T15:13:52,317 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0:(num 1731424400315) roll requested 2024-11-12T15:13:52,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741912_1088 (size=93) 2024-11-12T15:13:52,318 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta not finished, retry = 0 2024-11-12T15:13:52,329 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424432317, exclude list is [], retry=0 2024-11-12T15:13:52,330 DEBUG [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-12T15:13:52,330 INFO [rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1731424432317 on 2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33325,DS-e886ce3b-9173-4f04-a761-735839061db7,DISK] 2024-11-12T15:13:52,333 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424432317 2024-11-12T15:13:52,333 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37305:37305)] 2024-11-12T15:13:52,333 DEBUG [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 is not closed yet, will try archiving it next time 2024-11-12T15:13:52,333 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424367172 2024-11-12T15:13:52,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741913_1089 (size=653) 2024-11-12T15:13:52,335 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 not finished, retry = 0 2024-11-12T15:13:52,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-12T15:13:52,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:52,347 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-12T15:13:52,347 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '2b6d221c5cde,34071,1731424341329' in zk 2024-11-12T15:13:52,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,352 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-12T15:13:52,352 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:13:52,352 DEBUG [member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-12T15:13:52,352 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,352 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:52,352 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:52,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:52,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:52,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:52,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:52,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:52,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '2b6d221c5cde,34071,1731424341329': 2024-11-12T15:13:52,357 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-12T15:13:52,357 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '2b6d221c5cde,34071,1731424341329' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-12T15:13:52,357 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-12T15:13:52,357 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-12T15:13:52,358 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-12T15:13:52,358 INFO [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-12T15:13:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:13:52,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-12T15:13:52,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-12T15:13:52,373 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-12T15:13:52,374 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:13:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:13:52,374 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:52,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-12T15:13:52,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:52,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,375 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-12T15:13:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-12T15:13:52,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-12T15:13:52,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-12T15:13:52,384 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-12T15:13:52,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-12T15:13:52,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-12T15:13:52,385 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-12T15:13:52,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-12T15:13:52,388 DEBUG [(2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-12T15:13:52,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-12T15:13:52,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-12T15:13:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-12T15:13:52,397 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {2b6d221c5cde:34071=1731424386771} newestTimestamps: {2b6d221c5cde:34071=1731424400315} 2024-11-12T15:13:52,398 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 2b6d221c5cde%2C34071%2C1731424341329.meta.1731424345759.meta 2024-11-12T15:13:52,398 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 2b6d221c5cde%2C34071%2C1731424341329.meta.1731424386749.meta 2024-11-12T15:13:52,398 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta 2024-11-12T15:13:52,398 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 2b6d221c5cde%2C34071%2C1731424341329.meta.1731424432298.meta 2024-11-12T15:13:52,398 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 2024-11-12T15:13:52,399 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424432317 2024-11-12T15:13:52,399 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424400282 2024-11-12T15:13:52,399 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424432269 2024-11-12T15:13:52,399 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424367144.meta 2024-11-12T15:13:52,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741957_1133 (size=640) 2024-11-12T15:13:52,419 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424400299.meta 2024-11-12T15:13:52,436 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 2024-11-12T15:13:52,810 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:33269/backupUT/backup_1731424429639/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:52,810 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:52,813 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-12T15:13:52,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741958_1134 (size=45) 2024-11-12T15:13:53,218 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-12T15:13:53,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741959_1135 (size=390) 2024-11-12T15:13:53,636 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17314243507532 target: hdfs://localhost:33269/backupUT/backup_1731424429639/ns3/test-17314243507532 descriptor: 'ns3:test-17314243507532', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:53,636 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:53,639 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17314243507532 2024-11-12T15:13:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741960_1136 (size=50) 2024-11-12T15:13:54,046 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17314243507532 2024-11-12T15:13:54,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741961_1137 (size=950) 2024-11-12T15:13:54,457 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1731424350753 target: hdfs://localhost:33269/backupUT/backup_1731424429639/default/test-1731424350753 descriptor: 'test-1731424350753', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:54,457 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:54,460 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1731424350753 2024-11-12T15:13:54,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741962_1138 (size=53) 2024-11-12T15:13:54,866 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1731424350753 2024-11-12T15:13:54,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741963_1139 (size=364) 2024-11-12T15:13:55,277 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17314243507531 target: hdfs://localhost:33269/backupUT/backup_1731424429639/ns2/test-17314243507531 descriptor: 'ns2:test-17314243507531', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:55,277 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:55,280 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17314243507531 2024-11-12T15:13:55,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741964_1140 (size=50) 2024-11-12T15:13:55,686 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17314243507531 2024-11-12T15:13:55,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741965_1141 (size=390) 2024-11-12T15:13:55,698 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17314243507533 target: hdfs://localhost:33269/backupUT/backup_1731424429639/ns4/test-17314243507533 descriptor: 'ns4:test-17314243507533', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T15:13:55,699 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-12T15:13:55,701 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17314243507533 2024-11-12T15:13:55,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741966_1142 (size=50) 2024-11-12T15:13:56,106 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17314243507533 2024-11-12T15:13:56,118 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:33269/backupUT/.tmp/backup_1731424429639 from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315;hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424400282 2024-11-12T15:13:56,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542a1e5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:56,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:56,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:56,120 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:56,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:56,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:56,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1004dd50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:56,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:56,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:56,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:56,122 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:56,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c92dcc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:56,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T15:13:56,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,43085,1731424339322 2024-11-12T15:13:56,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c2a6bc5 2024-11-12T15:13:56,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T15:13:56,125 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T15:13:56,128 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-12T15:13:56,128 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-12T15:13:56,129 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-12T15:13:56,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:56,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:56,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:56,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:56,132 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-12T15:13:56,132 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1731424350753 2024-11-12T15:13:56,132 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1731424350753,, stopping at row=test-1731424350753 ,, for max=2147483647 with caching=100 2024-11-12T15:13:56,134 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1731424350753: test-1731424350753; 2024-11-12T15:13:56,134 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17314243507531 2024-11-12T15:13:56,134 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17314243507531,, stopping at row=ns2:test-17314243507531 ,, for max=2147483647 with caching=100 2024-11-12T15:13:56,136 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17314243507531: ns2:test-17314243507531; 2024-11-12T15:13:56,136 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17314243507532 2024-11-12T15:13:56,136 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17314243507532,, stopping at row=ns3:test-17314243507532 ,, for max=2147483647 with caching=100 2024-11-12T15:13:56,138 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17314243507532: ns3:test-17314243507532; 2024-11-12T15:13:56,138 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17314243507533 2024-11-12T15:13:56,138 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17314243507533,, stopping at row=ns4:test-17314243507533 ,, for max=2147483647 with caching=100 2024-11-12T15:13:56,139 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17314243507533: ns4:test-17314243507533; 2024-11-12T15:13:56,139 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-12T15:13:56,139 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/hbase-staging/partitions_ede35b69-6c2e-4a50-95ee-4beb163a9246 2024-11-12T15:13:56,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741967_1143 (size=324) 2024-11-12T15:13:56,546 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:56,547 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:56,547 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,533 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-13682057743323917745.jar 2024-11-12T15:13:57,534 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,534 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,535 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,535 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,535 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,535 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,536 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,536 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,536 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,537 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:57,537 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-12T15:13:57,537 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-12T15:13:57,537 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-12T15:13:57,538 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-12T15:13:57,538 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-12T15:13:57,538 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-12T15:13:57,539 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-12T15:13:57,539 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-12T15:13:57,539 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-12T15:13:57,539 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-12T15:13:57,540 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,540 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,540 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,540 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,541 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,541 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-12T15:13:57,541 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-12T15:13:57,541 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,default:test-1731424350753,ns2:test-17314243507531,ns3:test-17314243507532,ns4:test-17314243507533 2024-11-12T15:13:57,542 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:13:57,542 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:13:57,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:57,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:57,542 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:57,543 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-12T15:13:58,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop-11729306893628489712.jar 2024-11-12T15:13:58,642 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-12T15:13:58,670 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 for WAL files 2024-11-12T15:13:58,671 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1731424429639File hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 does not exist. java.io.FileNotFoundException: File hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T15:13:58,672 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1731424429639,startts=1731424432034,failedts=1731424438672,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0.1731424400315 does not exist. 2024-11-12T15:13:58,672 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-12T15:13:58,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable backup:system 2024-11-12T15:13:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-12T15:13:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T15:13:58,692 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424438691"}]},"ts":"1731424438691"} 2024-11-12T15:13:58,694 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-12T15:13:58,694 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-12T15:13:58,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-12T15:13:58,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, UNASSIGN}] 2024-11-12T15:13:58,704 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, UNASSIGN 2024-11-12T15:13:58,705 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9ce3fb3bd1b279fea0145fdedac19450, regionState=CLOSING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:13:58,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, UNASSIGN because future has completed 2024-11-12T15:13:58,708 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T15:13:58,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:13:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T15:13:58,862 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:58,862 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T15:13:58,862 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 9ce3fb3bd1b279fea0145fdedac19450, disabling compactions & flushes 2024-11-12T15:13:58,862 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:58,862 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:58,863 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. after waiting 0 ms 2024-11-12T15:13:58,863 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:58,863 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 9ce3fb3bd1b279fea0145fdedac19450 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-12T15:13:58,867 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/efa009de324643ba8d25dad634ea30e0 is 103, key is rslogts:hdfs://localhost:33269/backupUT\x002b6d221c5cde:34071/meta:rs-log-ts/1731424432346/Put/seqid=0 2024-11-12T15:13:58,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741968_1144 (size=5338) 2024-11-12T15:13:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T15:13:59,276 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/efa009de324643ba8d25dad634ea30e0 2024-11-12T15:13:59,284 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/67ed21b75111414fade37a13517b513a is 303, key is session:backup_1731424429639/session:context/1731424432035/Put/seqid=0 2024-11-12T15:13:59,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741969_1145 (size=5391) 2024-11-12T15:13:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T15:13:59,689 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/67ed21b75111414fade37a13517b513a 2024-11-12T15:13:59,696 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/efa009de324643ba8d25dad634ea30e0 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/efa009de324643ba8d25dad634ea30e0 2024-11-12T15:13:59,705 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/efa009de324643ba8d25dad634ea30e0, entries=1, sequenceid=47, filesize=5.2 K 2024-11-12T15:13:59,706 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/67ed21b75111414fade37a13517b513a as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/67ed21b75111414fade37a13517b513a 2024-11-12T15:13:59,713 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/67ed21b75111414fade37a13517b513a, entries=1, sequenceid=47, filesize=5.3 K 2024-11-12T15:13:59,714 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 851ms, sequenceid=47, compaction requested=true 2024-11-12T15:13:59,719 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-12T15:13:59,720 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:13:59,720 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:13:59,720 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 9ce3fb3bd1b279fea0145fdedac19450: Waiting for close lock at 1731424438862Running coprocessor pre-close hooks at 1731424438862Disabling compacts and flushes for region at 1731424438862Disabling writes for close at 1731424438863 (+1 ms)Obtaining lock to block concurrent updates at 1731424438863Preparing flush snapshotting stores in 9ce3fb3bd1b279fea0145fdedac19450 at 1731424438863Finished memstore snapshotting backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1731424438863Flushing stores of backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. at 1731424438864 (+1 ms)Flushing 9ce3fb3bd1b279fea0145fdedac19450/meta: creating writer at 1731424438864Flushing 9ce3fb3bd1b279fea0145fdedac19450/meta: appending metadata at 1731424438867 (+3 ms)Flushing 9ce3fb3bd1b279fea0145fdedac19450/meta: closing flushed file at 1731424438867Flushing 9ce3fb3bd1b279fea0145fdedac19450/session: creating writer at 1731424439281 (+414 ms)Flushing 9ce3fb3bd1b279fea0145fdedac19450/session: appending metadata at 1731424439284 (+3 ms)Flushing 9ce3fb3bd1b279fea0145fdedac19450/session: closing flushed file at 1731424439284Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@88a8f8b: reopening flushed file at 1731424439695 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d5af25e: reopening flushed file at 1731424439705 (+10 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 851ms, sequenceid=47, compaction requested=true at 1731424439714 (+9 ms)Writing region close event to WAL at 1731424439715 (+1 ms)Running coprocessor post-close hooks at 1731424439720 (+5 ms)Closed at 1731424439720 2024-11-12T15:13:59,722 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:13:59,723 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9ce3fb3bd1b279fea0145fdedac19450, regionState=CLOSED 2024-11-12T15:13:59,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:13:59,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T15:13:59,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329 in 1.0170 sec 2024-11-12T15:13:59,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-12T15:13:59,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, UNASSIGN in 1.0240 sec 2024-11-12T15:13:59,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T15:13:59,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0330 sec 2024-11-12T15:13:59,733 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424439732"}]},"ts":"1731424439732"} 2024-11-12T15:13:59,735 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-12T15:13:59,735 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-12T15:13:59,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0530 sec 2024-11-12T15:13:59,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-12T15:13:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T15:13:59,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-12T15:13:59,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1731424439833 2024-11-12T15:13:59,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731424439834 (current time:1731424439834). 2024-11-12T15:13:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-12T15:13:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1731424439833 VERSION not specified, setting to 2 2024-11-12T15:13:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d21846f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,43085,-1 for getting cluster id 2024-11-12T15:13:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T15:13:59,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c1f44d9-3bb0-42bc-892d-04aff690dcc3' 2024-11-12T15:13:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T15:13:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c1f44d9-3bb0-42bc-892d-04aff690dcc3" 2024-11-12T15:13:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145fab82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,43085,-1] 2024-11-12T15:13:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T15:13:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:59,837 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T15:13:59,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec6ae1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-12T15:13:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T15:13:59,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,34071,1731424341329, seqNum=-1] 2024-11-12T15:13:59,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T15:13:59,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T15:13:59,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085. 2024-11-12T15:13:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T15:13:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:13:59,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:13:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-12T15:13:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:13:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-12T15:13:59,844 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-12T15:13:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T15:13:59,846 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-12T15:13:59,849 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-12T15:13:59,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741970_1146 (size=81) 2024-11-12T15:13:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T15:14:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T15:14:00,260 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-12T15:14:00,263 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-12T15:14:00,263 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-12T15:14:00,263 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:14:00,267 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/efa009de324643ba8d25dad634ea30e0] hfiles 2024-11-12T15:14:00,267 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 2024-11-12T15:14:00,267 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0 2024-11-12T15:14:00,267 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 2024-11-12T15:14:00,268 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/efa009de324643ba8d25dad634ea30e0 2024-11-12T15:14:00,268 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:14:00,271 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/67ed21b75111414fade37a13517b513a, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51] hfiles 2024-11-12T15:14:00,271 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733 2024-11-12T15:14:00,271 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d 2024-11-12T15:14:00,271 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/67ed21b75111414fade37a13517b513a 2024-11-12T15:14:00,271 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed 2024-11-12T15:14:00,271 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 2024-11-12T15:14:00,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741971_1147 (size=409) 2024-11-12T15:14:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T15:14:00,679 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 9ce3fb3bd1b279fea0145fdedac19450, NAME => 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-12T15:14:00,682 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-12T15:14:00,683 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-12T15:14:00,683 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1731424439833 2024-11-12T15:14:00,684 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1731424439833 2024-11-12T15:14:00,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741972_1148 (size=1046) 2024-11-12T15:14:00,794 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e4f31acd4066dc4cd42e4f7b965c9073, had cached 0 bytes from a total of 38453 2024-11-12T15:14:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T15:14:01,101 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-12T15:14:01,116 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-12T15:14:01,116 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1731424439833 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1731424439833 2024-11-12T15:14:01,118 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-12T15:14:01,118 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-12T15:14:01,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1731424439833 table=backup:system type=FLUSH ttl=0 } in 1.2760 sec 2024-11-12T15:14:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T15:14:02,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-12T15:14:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731424429868 type: FLUSH version: 2 ttl: 0 ) 2024-11-12T15:14:02,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-12T15:14:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T15:14:02,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741973_1149 (size=635) 2024-11-12T15:14:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T15:14:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T15:14:02,447 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-12T15:14:02,447 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-12T15:14:02,449 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-12T15:14:02,454 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731424429868 type: FLUSH version: 2 ttl: 0 2024-11-12T15:14:02,455 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system 2024-11-12T15:14:02,458 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-12T15:14:02,458 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(229): region to restore: 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:02,462 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:14:02,465 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/efa009de324643ba8d25dad634ea30e0 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/efa009de324643ba8d25dad634ea30e0 2024-11-12T15:14:02,465 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:14:02,468 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/67ed21b75111414fade37a13517b513a to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/67ed21b75111414fade37a13517b513a 2024-11-12T15:14:02,469 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731424429868 type: FLUSH version: 2 ttl: 0 2024-11-12T15:14:02,470 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731424442470"}]},"ts":"9223372036854775807"} 2024-11-12T15:14:02,474 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T15:14:02,474 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9ce3fb3bd1b279fea0145fdedac19450, NAME => 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T15:14:02,474 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1731424442471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731424442471"}]},"ts":"1731424442471"} 2024-11-12T15:14:02,476 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T15:14:02,476 INFO [PEWorker-5 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-12T15:14:02,476 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => 9ce3fb3bd1b279fea0145fdedac19450, NAME => 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T15:14:02,477 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-12T15:14:02,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1731424429868 type: FLUSH version: 2 ttl: 0 ) in 452 msec 2024-11-12T15:14:02,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-12T15:14:02,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1731424439833 2024-11-12T15:14:02,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "hbase-failsafe-snapshot_backup_system-1731424439833" type: DISABLED 2024-11-12T15:14:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1731424439833 2024-11-12T15:14:02,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.3 enable backup:system 2024-11-12T15:14:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-12T15:14:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T15:14:02,678 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-12T15:14:02,678 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424442678"}]},"ts":"1731424442678"} 2024-11-12T15:14:02,680 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-12T15:14:02,681 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-12T15:14:02,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN}] 2024-11-12T15:14:02,682 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN 2024-11-12T15:14:02,683 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-12T15:14:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T15:14:02,834 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=9ce3fb3bd1b279fea0145fdedac19450, regionState=OPENING, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:14:02,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN because future has completed 2024-11-12T15:14:02,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329}] 2024-11-12T15:14:02,992 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:02,992 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ce3fb3bd1b279fea0145fdedac19450, NAME => 'backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.', STARTKEY => '', ENDKEY => ''} 2024-11-12T15:14:02,992 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-12T15:14:02,992 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:02,992 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T15:14:02,993 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:02,993 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T15:14:02,994 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:02,995 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ce3fb3bd1b279fea0145fdedac19450 columnFamilyName meta 2024-11-12T15:14:02,995 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:14:03,003 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 2024-11-12T15:14:03,009 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0 2024-11-12T15:14:03,015 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 2024-11-12T15:14:03,015 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(327): Store=9ce3fb3bd1b279fea0145fdedac19450/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:14:03,016 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,017 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ce3fb3bd1b279fea0145fdedac19450 columnFamilyName session 2024-11-12T15:14:03,017 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T15:14:03,025 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733 2024-11-12T15:14:03,031 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d 2024-11-12T15:14:03,035 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed 2024-11-12T15:14:03,040 DEBUG [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 2024-11-12T15:14:03,040 INFO [StoreOpener-9ce3fb3bd1b279fea0145fdedac19450-1 {}] regionserver.HStore(327): Store=9ce3fb3bd1b279fea0145fdedac19450/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T15:14:03,041 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,042 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,043 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,044 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,044 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,044 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-12T15:14:03,046 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,046 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened 9ce3fb3bd1b279fea0145fdedac19450; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10177881440, jitterRate=-0.05211092531681061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-12T15:14:03,046 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,047 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for 9ce3fb3bd1b279fea0145fdedac19450: Running coprocessor pre-open hook at 1731424442993Writing region info on filesystem at 1731424442993Initializing all the Stores at 1731424442994 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424442994Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731424442994Cleaning up temporary data from old regions at 1731424443044 (+50 ms)Running coprocessor post-open hooks at 1731424443047 (+3 ms)Region opened successfully at 1731424443047 2024-11-12T15:14:03,048 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., pid=70, masterSystemTime=1731424442988 2024-11-12T15:14:03,049 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 9ce3fb3bd1b279fea0145fdedac19450:meta, priority=-2147483648, current under compaction store size is 1 2024-11-12T15:14:03,051 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T15:14:03,051 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T15:14:03,052 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 9ce3fb3bd1b279fea0145fdedac19450:session, priority=-2147483648, current under compaction store size is 2 2024-11-12T15:14:03,052 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T15:14:03,052 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T15:14:03,054 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T15:14:03,054 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T15:14:03,054 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,054 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,056 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.HStore(1541): 9ce3fb3bd1b279fea0145fdedac19450/session is initiating minor compaction (all files) 2024-11-12T15:14:03,056 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.HStore(1541): 9ce3fb3bd1b279fea0145fdedac19450/meta is initiating minor compaction (all files) 2024-11-12T15:14:03,056 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=9ce3fb3bd1b279fea0145fdedac19450, regionState=OPEN, openSeqNum=51, regionLocation=2b6d221c5cde,34071,1731424341329 2024-11-12T15:14:03,057 INFO [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ce3fb3bd1b279fea0145fdedac19450/session in backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,057 INFO [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9ce3fb3bd1b279fea0145fdedac19450/meta in backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,057 INFO [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0] into tmpdir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp, totalSize=17.9 K 2024-11-12T15:14:03,057 INFO [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733] into tmpdir=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp, totalSize=22.7 K 2024-11-12T15:14:03,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329 because future has completed 2024-11-12T15:14:03,058 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1e985d0d7044c1972e4dbf0dba516d, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731424364728 2024-11-12T15:14:03,058 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0c644b692448d39ced11a9e55b9981, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731424367293 2024-11-12T15:14:03,059 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] compactions.Compactor(225): Compacting 74df3208bc1741599a024776009497ed, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731424379879 2024-11-12T15:14:03,060 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e66fcb0119e4f0eb7f92f926a08c5d0, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731424386844 2024-11-12T15:14:03,060 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] compactions.Compactor(225): Compacting ece27341f33b4851ad44a3c1b49adb51, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731424394842 2024-11-12T15:14:03,060 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59310573775849a5a04697e1a3a246c0, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731424400361 2024-11-12T15:14:03,061 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] compactions.Compactor(225): Compacting 14347bca6aca4942922ff4c9589be733, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731424424602 2024-11-12T15:14:03,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-12T15:14:03,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ce3fb3bd1b279fea0145fdedac19450, server=2b6d221c5cde,34071,1731424341329 in 223 msec 2024-11-12T15:14:03,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T15:14:03,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=9ce3fb3bd1b279fea0145fdedac19450, ASSIGN in 382 msec 2024-11-12T15:14:03,068 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731424443068"}]},"ts":"1731424443068"} 2024-11-12T15:14:03,073 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-12T15:14:03,073 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-12T15:14:03,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 402 msec 2024-11-12T15:14:03,106 INFO [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ce3fb3bd1b279fea0145fdedac19450#meta#compaction#16 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T15:14:03,107 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/a7260773cd1243f5a0a34e70209a5cca is 158, key is trslm:hdfs://localhost:33269/backupUT\x00ns2:test-17314243507531/meta:log-roll-map/1731424424159/Put/seqid=0 2024-11-12T15:14:03,114 INFO [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9ce3fb3bd1b279fea0145fdedac19450#session#compaction#17 average throughput is 2.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T15:14:03,115 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/34a8d3d18eae4eb3a049b8bddd25a527 is 1245, key is session:backup_1731424395864/session:context/1731424395965/Put/seqid=0 2024-11-12T15:14:03,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741974_1150 (size=6749) 2024-11-12T15:14:03,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741975_1151 (size=8073) 2024-11-12T15:14:03,128 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/34a8d3d18eae4eb3a049b8bddd25a527 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/34a8d3d18eae4eb3a049b8bddd25a527 2024-11-12T15:14:03,140 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-12T15:14:03,142 INFO [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9ce3fb3bd1b279fea0145fdedac19450/session of 9ce3fb3bd1b279fea0145fdedac19450 into 34a8d3d18eae4eb3a049b8bddd25a527(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T15:14:03,142 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ce3fb3bd1b279fea0145fdedac19450: 2024-11-12T15:14:03,145 INFO [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., storeName=9ce3fb3bd1b279fea0145fdedac19450/session, priority=12, startTime=1731424443051; duration=0sec 2024-11-12T15:14:03,146 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T15:14:03,146 DEBUG [RS:0;2b6d221c5cde:34071-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ce3fb3bd1b279fea0145fdedac19450:session 2024-11-12T15:14:03,254 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0e9acca221b224195ce9cae6845295fe, had cached 0 bytes from a total of 13547 2024-11-12T15:14:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T15:14:03,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-12T15:14:03,305 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-12T15:14:03,305 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-12T15:14:03,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-11-12T15:14:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43085 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-12T15:14:03,319 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-12T15:14:03,319 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-12T15:14:03,322 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-12T15:14:03,328 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1731424429639 failed. 2024-11-12T15:14:03,328 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:14:03,328 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:14:03,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:03,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:03,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:14:03,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=626 (was 611) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;2b6d221c5cde:34071-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1953189042-172.17.0.3-1731424333465:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1637559001_20 at /127.0.0.1:35408 [Receiving block BP-1953189042-172.17.0.3-1731424333465:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1953189042-172.17.0.3-1731424333465:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: '2b6d221c5cde,34071,1731424341329' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1637559001_20 at /127.0.0.1:35426 [Receiving block BP-1953189042-172.17.0.3-1731424333465:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (2b6d221c5cde,43085,1731424339322)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-321164338_20 at /127.0.0.1:50620 [Waiting for operation #75] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1637559001_20 at /127.0.0.1:35424 [Receiving block BP-1953189042-172.17.0.3-1731424333465:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/a7260773cd1243f5a0a34e70209a5cca java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(2b6d221c5cde,34071,1731424341329)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1953189042-172.17.0.3-1731424333465:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=639 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 464) - SystemLoadAverage LEAK? -, ProcessCount=9 (was 9), AvailableMemoryMB=2298 (was 4708) 2024-11-12T15:14:03,353 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=626 is superior to 500 2024-11-12T15:14:03,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T15:14:03,360 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:14:03,360 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:14:03,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:03,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:03,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:14:03,360 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T15:14:03,360 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=469716529, stopped=false 2024-11-12T15:14:03,361 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2b6d221c5cde,43085,1731424339322 2024-11-12T15:14:03,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-12T15:14:03,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-12T15:14:03,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:14:03,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:14:03,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T15:14:03,416 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T15:14:03,416 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:14:03,417 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T15:14:03,417 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-12T15:14:03,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:03,419 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2b6d221c5cde,34071,1731424341329' ***** 2024-11-12T15:14:03,419 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T15:14:03,419 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-12T15:14:03,425 INFO [RS:0;2b6d221c5cde:34071 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@7253afc7{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-12T15:14:03,427 INFO [RS:0;2b6d221c5cde:34071 {}] server.AbstractConnector(383): Stopped ServerConnector@3b99b3c7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-12T15:14:03,427 INFO [RS:0;2b6d221c5cde:34071 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:03,427 INFO [RS:0;2b6d221c5cde:34071 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@126a703e{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-12T15:14:03,427 INFO [RS:0;2b6d221c5cde:34071 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@21ae8981{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:03,429 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T15:14:03,429 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T15:14:03,429 INFO [RS:0;2b6d221c5cde:34071 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T15:14:03,429 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-12T15:14:03,429 INFO [RS:0;2b6d221c5cde:34071 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T15:14:03,429 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for e4f31acd4066dc4cd42e4f7b965c9073 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for 0e9acca221b224195ce9cae6845295fe 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for 9ce3fb3bd1b279fea0145fdedac19450 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for aa6d9848b308420055c29add59789661 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for b64e86ccc223899acbc0b981c82d38b4 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:14:03,430 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e4f31acd4066dc4cd42e4f7b965c9073, disabling compactions & flushes 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(959): stopping server 2b6d221c5cde,34071,1731424341329 2024-11-12T15:14:03,430 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:14:03,430 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T15:14:03,430 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. after waiting 0 ms 2024-11-12T15:14:03,430 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:14:03,430 INFO [RS:0;2b6d221c5cde:34071 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2b6d221c5cde:34071. 2024-11-12T15:14:03,430 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e4f31acd4066dc4cd42e4f7b965c9073 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-12T15:14:03,430 DEBUG [RS:0;2b6d221c5cde:34071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:14:03,430 DEBUG [RS:0;2b6d221c5cde:34071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:03,431 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T15:14:03,431 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T15:14:03,444 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f2/35a62f0f156e497eb5089c461d4e0170 is 58, key is row-test-1731424350753-2-0/f2:q1/1731424424629/Put/seqid=0 2024-11-12T15:14:03,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741976_1152 (size=5464) 2024-11-12T15:14:03,491 INFO [regionserver/2b6d221c5cde:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T15:14:03,530 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/meta/a7260773cd1243f5a0a34e70209a5cca as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/a7260773cd1243f5a0a34e70209a5cca 2024-11-12T15:14:03,538 INFO [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9ce3fb3bd1b279fea0145fdedac19450/meta of 9ce3fb3bd1b279fea0145fdedac19450 into a7260773cd1243f5a0a34e70209a5cca(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T15:14:03,538 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9ce3fb3bd1b279fea0145fdedac19450: 2024-11-12T15:14:03,539 INFO [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., storeName=9ce3fb3bd1b279fea0145fdedac19450/meta, priority=13, startTime=1731424443049; duration=0sec 2024-11-12T15:14:03,539 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T15:14:03,539 DEBUG [RS:0;2b6d221c5cde:34071-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9ce3fb3bd1b279fea0145fdedac19450:meta 2024-11-12T15:14:03,539 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T15:14:03,539 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T15:14:03,540 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-12T15:14:03,540 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T15:14:03,540 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1325): Online Regions={e4f31acd4066dc4cd42e4f7b965c9073=test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., 1588230740=hbase:meta,,1.1588230740, 0e9acca221b224195ce9cae6845295fe=ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe., 9ce3fb3bd1b279fea0145fdedac19450=backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., aa6d9848b308420055c29add59789661=ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661., b64e86ccc223899acbc0b981c82d38b4=ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4., fb8c7f767a06180d5cc67ec654094c61=backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61.} 2024-11-12T15:14:03,540 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T15:14:03,540 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T15:14:03,540 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T15:14:03,540 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T15:14:03,540 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-12T15:14:03,540 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 0e9acca221b224195ce9cae6845295fe, 1588230740, 9ce3fb3bd1b279fea0145fdedac19450, aa6d9848b308420055c29add59789661, b64e86ccc223899acbc0b981c82d38b4, e4f31acd4066dc4cd42e4f7b965c9073, fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:14:03,560 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/info/7ad731814e8e46279d8b391fdd4e0fbb is 159, key is ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe./info:regioninfo/1731424353281/Put/seqid=0 2024-11-12T15:14:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741977_1153 (size=14126) 2024-11-12T15:14:03,740 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 0e9acca221b224195ce9cae6845295fe, 1588230740, 9ce3fb3bd1b279fea0145fdedac19450, aa6d9848b308420055c29add59789661, b64e86ccc223899acbc0b981c82d38b4, e4f31acd4066dc4cd42e4f7b965c9073, fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:14:03,849 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f2/35a62f0f156e497eb5089c461d4e0170 2024-11-12T15:14:03,860 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/.tmp/f2/35a62f0f156e497eb5089c461d4e0170 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f2/35a62f0f156e497eb5089c461d4e0170 2024-11-12T15:14:03,867 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/f2/35a62f0f156e497eb5089c461d4e0170, entries=7, sequenceid=348, filesize=5.3 K 2024-11-12T15:14:03,868 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for e4f31acd4066dc4cd42e4f7b965c9073 in 438ms, sequenceid=348, compaction requested=false 2024-11-12T15:14:03,873 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/default/test-1731424350753/e4f31acd4066dc4cd42e4f7b965c9073/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-12T15:14:03,874 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:03,874 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:14:03,874 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e4f31acd4066dc4cd42e4f7b965c9073: Waiting for close lock at 1731424443430Running coprocessor pre-close hooks at 1731424443430Disabling compacts and flushes for region at 1731424443430Disabling writes for close at 1731424443430Obtaining lock to block concurrent updates at 1731424443430Preparing flush snapshotting stores in e4f31acd4066dc4cd42e4f7b965c9073 at 1731424443430Finished memstore snapshotting test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1731424443431 (+1 ms)Flushing stores of test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. at 1731424443431Flushing e4f31acd4066dc4cd42e4f7b965c9073/f2: creating writer at 1731424443431Flushing e4f31acd4066dc4cd42e4f7b965c9073/f2: appending metadata at 1731424443444 (+13 ms)Flushing e4f31acd4066dc4cd42e4f7b965c9073/f2: closing flushed file at 1731424443444Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72848e17: reopening flushed file at 1731424443859 (+415 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for e4f31acd4066dc4cd42e4f7b965c9073 in 438ms, sequenceid=348, compaction requested=false at 1731424443868 (+9 ms)Writing region close event to WAL at 1731424443870 (+2 ms)Running coprocessor post-close hooks at 1731424443874 (+4 ms)Closed at 1731424443874 2024-11-12T15:14:03,874 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1731424350753,,1731424351231.e4f31acd4066dc4cd42e4f7b965c9073. 2024-11-12T15:14:03,874 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0e9acca221b224195ce9cae6845295fe, disabling compactions & flushes 2024-11-12T15:14:03,875 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:14:03,875 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:14:03,875 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. after waiting 0 ms 2024-11-12T15:14:03,875 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:14:03,879 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns2/test-17314243507531/0e9acca221b224195ce9cae6845295fe/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-12T15:14:03,879 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:03,879 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:14:03,879 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0e9acca221b224195ce9cae6845295fe: Waiting for close lock at 1731424443874Running coprocessor pre-close hooks at 1731424443874Disabling compacts and flushes for region at 1731424443874Disabling writes for close at 1731424443875 (+1 ms)Writing region close event to WAL at 1731424443875Running coprocessor post-close hooks at 1731424443879 (+4 ms)Closed at 1731424443879 2024-11-12T15:14:03,879 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17314243507531,,1731424352091.0e9acca221b224195ce9cae6845295fe. 2024-11-12T15:14:03,879 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9ce3fb3bd1b279fea0145fdedac19450, disabling compactions & flushes 2024-11-12T15:14:03,879 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,880 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,880 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. after waiting 0 ms 2024-11-12T15:14:03,880 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:03,880 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9ce3fb3bd1b279fea0145fdedac19450 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-12T15:14:03,883 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/1db612b4cbe34bc7bffd4730b3617f0c is 554, key is session:backup_1731424429639/session:context/1731424443321/Put/seqid=0 2024-11-12T15:14:03,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741978_1154 (size=5655) 2024-11-12T15:14:03,941 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9ce3fb3bd1b279fea0145fdedac19450, aa6d9848b308420055c29add59789661, b64e86ccc223899acbc0b981c82d38b4, fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:14:03,966 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/info/7ad731814e8e46279d8b391fdd4e0fbb 2024-11-12T15:14:03,991 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/ns/64a078e6b7d84febaef84fc6a6669187 is 43, key is default/ns:d/1731424345993/Put/seqid=0 2024-11-12T15:14:03,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741979_1155 (size=5345) 2024-11-12T15:14:04,141 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9ce3fb3bd1b279fea0145fdedac19450, aa6d9848b308420055c29add59789661, b64e86ccc223899acbc0b981c82d38b4, fb8c7f767a06180d5cc67ec654094c61 2024-11-12T15:14:04,223 INFO [regionserver/2b6d221c5cde:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T15:14:04,224 INFO [regionserver/2b6d221c5cde:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T15:14:04,288 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/1db612b4cbe34bc7bffd4730b3617f0c 2024-11-12T15:14:04,294 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/.tmp/session/1db612b4cbe34bc7bffd4730b3617f0c as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/1db612b4cbe34bc7bffd4730b3617f0c 2024-11-12T15:14:04,299 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/1db612b4cbe34bc7bffd4730b3617f0c, entries=2, sequenceid=57, filesize=5.5 K 2024-11-12T15:14:04,300 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 420ms, sequenceid=57, compaction requested=false 2024-11-12T15:14:04,300 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0] to archive 2024-11-12T15:14:04,301 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T15:14:04,305 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/ae0c644b692448d39ced11a9e55b9981 2024-11-12T15:14:04,306 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/0e66fcb0119e4f0eb7f92f926a08c5d0 2024-11-12T15:14:04,308 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/meta/59310573775849a5a04697e1a3a246c0 2024-11-12T15:14:04,313 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2b6d221c5cde:43085 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-12T15:14:04,313 WARN [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ae0c644b692448d39ced11a9e55b9981=5914, 0e66fcb0119e4f0eb7f92f926a08c5d0=5745, 59310573775849a5a04697e1a3a246c0=6646] 2024-11-12T15:14:04,314 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733] to archive 2024-11-12T15:14:04,314 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T15:14:04,316 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/2d1e985d0d7044c1972e4dbf0dba516d 2024-11-12T15:14:04,318 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/74df3208bc1741599a024776009497ed 2024-11-12T15:14:04,319 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/ece27341f33b4851ad44a3c1b49adb51 2024-11-12T15:14:04,320 DEBUG [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733 to hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/archive/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/session/14347bca6aca4942922ff4c9589be733 2024-11-12T15:14:04,320 WARN [StoreCloser-backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [2d1e985d0d7044c1972e4dbf0dba516d=5049, 74df3208bc1741599a024776009497ed=5535, ece27341f33b4851ad44a3c1b49adb51=6682, 14347bca6aca4942922ff4c9589be733=5928] 2024-11-12T15:14:04,323 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system/9ce3fb3bd1b279fea0145fdedac19450/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:04,324 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9ce3fb3bd1b279fea0145fdedac19450: Waiting for close lock at 1731424443879Running coprocessor pre-close hooks at 1731424443879Disabling compacts and flushes for region at 1731424443879Disabling writes for close at 1731424443880 (+1 ms)Obtaining lock to block concurrent updates at 1731424443880Preparing flush snapshotting stores in 9ce3fb3bd1b279fea0145fdedac19450 at 1731424443880Finished memstore snapshotting backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1731424443880Flushing stores of backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. at 1731424443880Flushing 9ce3fb3bd1b279fea0145fdedac19450/session: creating writer at 1731424443881 (+1 ms)Flushing 9ce3fb3bd1b279fea0145fdedac19450/session: appending metadata at 1731424443883 (+2 ms)Flushing 9ce3fb3bd1b279fea0145fdedac19450/session: closing flushed file at 1731424443883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dd00d78: reopening flushed file at 1731424444293 (+410 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 9ce3fb3bd1b279fea0145fdedac19450 in 420ms, sequenceid=57, compaction requested=false at 1731424444300 (+7 ms)Writing region close event to WAL at 1731424444320 (+20 ms)Running coprocessor post-close hooks at 1731424444323 (+3 ms)Closed at 1731424444324 (+1 ms) 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1731424360330.9ce3fb3bd1b279fea0145fdedac19450. 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa6d9848b308420055c29add59789661, disabling compactions & flushes 2024-11-12T15:14:04,324 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. after waiting 0 ms 2024-11-12T15:14:04,324 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:14:04,327 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns3/test-17314243507532/aa6d9848b308420055c29add59789661/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:04,328 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa6d9848b308420055c29add59789661: Waiting for close lock at 1731424444324Running coprocessor pre-close hooks at 1731424444324Disabling compacts and flushes for region at 1731424444324Disabling writes for close at 1731424444324Writing region close event to WAL at 1731424444324Running coprocessor post-close hooks at 1731424444328 (+4 ms)Closed at 1731424444328 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17314243507532,,1731424354512.aa6d9848b308420055c29add59789661. 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b64e86ccc223899acbc0b981c82d38b4, disabling compactions & flushes 2024-11-12T15:14:04,328 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. after waiting 0 ms 2024-11-12T15:14:04,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:14:04,332 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/ns4/test-17314243507533/b64e86ccc223899acbc0b981c82d38b4/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T15:14:04,332 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:04,332 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:14:04,332 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b64e86ccc223899acbc0b981c82d38b4: Waiting for close lock at 1731424444328Running coprocessor pre-close hooks at 1731424444328Disabling compacts and flushes for region at 1731424444328Disabling writes for close at 1731424444328Writing region close event to WAL at 1731424444329 (+1 ms)Running coprocessor post-close hooks at 1731424444332 (+3 ms)Closed at 1731424444332 2024-11-12T15:14:04,332 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17314243507533,,1731424356675.b64e86ccc223899acbc0b981c82d38b4. 2024-11-12T15:14:04,333 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb8c7f767a06180d5cc67ec654094c61, disabling compactions & flushes 2024-11-12T15:14:04,333 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:14:04,333 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:14:04,333 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. after waiting 0 ms 2024-11-12T15:14:04,333 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:14:04,336 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/backup/system_bulk/fb8c7f767a06180d5cc67ec654094c61/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-12T15:14:04,336 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:04,336 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:14:04,336 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb8c7f767a06180d5cc67ec654094c61: Waiting for close lock at 1731424444333Running coprocessor pre-close hooks at 1731424444333Disabling compacts and flushes for region at 1731424444333Disabling writes for close at 1731424444333Writing region close event to WAL at 1731424444333Running coprocessor post-close hooks at 1731424444336 (+3 ms)Closed at 1731424444336 2024-11-12T15:14:04,336 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1731424362511.fb8c7f767a06180d5cc67ec654094c61. 2024-11-12T15:14:04,341 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T15:14:04,395 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/ns/64a078e6b7d84febaef84fc6a6669187 2024-11-12T15:14:04,419 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/table/122e79d05c154251b8a3a8828bcf937b is 59, key is ns2:test-17314243507531/table:state/1731424353297/Put/seqid=0 2024-11-12T15:14:04,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741980_1156 (size=5866) 2024-11-12T15:14:04,542 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T15:14:04,542 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T15:14:04,542 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T15:14:04,742 DEBUG [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T15:14:04,824 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/table/122e79d05c154251b8a3a8828bcf937b 2024-11-12T15:14:04,830 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/info/7ad731814e8e46279d8b391fdd4e0fbb as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/info/7ad731814e8e46279d8b391fdd4e0fbb 2024-11-12T15:14:04,836 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/info/7ad731814e8e46279d8b391fdd4e0fbb, entries=69, sequenceid=59, filesize=13.8 K 2024-11-12T15:14:04,837 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/ns/64a078e6b7d84febaef84fc6a6669187 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/ns/64a078e6b7d84febaef84fc6a6669187 2024-11-12T15:14:04,843 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/ns/64a078e6b7d84febaef84fc6a6669187, entries=7, sequenceid=59, filesize=5.2 K 2024-11-12T15:14:04,844 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/.tmp/table/122e79d05c154251b8a3a8828bcf937b as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/table/122e79d05c154251b8a3a8828bcf937b 2024-11-12T15:14:04,850 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/table/122e79d05c154251b8a3a8828bcf937b, entries=13, sequenceid=59, filesize=5.7 K 2024-11-12T15:14:04,851 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1311ms, sequenceid=59, compaction requested=false 2024-11-12T15:14:04,860 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-12T15:14:04,861 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-12T15:14:04,861 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T15:14:04,861 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T15:14:04,861 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731424443540Running coprocessor pre-close hooks at 1731424443540Disabling compacts and flushes for region at 1731424443540Disabling writes for close at 1731424443540Obtaining lock to block concurrent updates at 1731424443540Preparing flush snapshotting stores in 1588230740 at 1731424443540Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1731424443540Flushing stores of hbase:meta,,1.1588230740 at 1731424443541 (+1 ms)Flushing 1588230740/info: creating writer at 1731424443541Flushing 1588230740/info: appending metadata at 1731424443560 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731424443560Flushing 1588230740/ns: creating writer at 1731424443975 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731424443990 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731424443990Flushing 1588230740/table: creating writer at 1731424444403 (+413 ms)Flushing 1588230740/table: appending metadata at 1731424444419 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731424444419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5beaa9ae: reopening flushed file at 1731424444829 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f8c43c5: reopening flushed file at 1731424444836 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3864ba76: reopening flushed file at 1731424444843 (+7 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1311ms, sequenceid=59, compaction requested=false at 1731424444851 (+8 ms)Writing region close event to WAL at 1731424444856 (+5 ms)Running coprocessor post-close hooks at 1731424444861 (+5 ms)Closed at 1731424444861 2024-11-12T15:14:04,861 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T15:14:04,942 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(976): stopping server 2b6d221c5cde,34071,1731424341329; all regions closed. 2024-11-12T15:14:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741955_1131 (size=3260) 2024-11-12T15:14:04,946 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.meta.1731424432298.meta not finished, retry = 0 2024-11-12T15:14:05,051 DEBUG [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs 2024-11-12T15:14:05,051 INFO [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.meta:.meta(num 1731424432298) 2024-11-12T15:14:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741956_1132 (size=1037) 2024-11-12T15:14:05,055 DEBUG [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs 2024-11-12T15:14:05,056 INFO [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-0:(num 1731424432317) 2024-11-12T15:14:05,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741954_1130 (size=5275) 2024-11-12T15:14:05,058 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/WALs/2b6d221c5cde,34071,1731424341329/2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1.1731424432269 not finished, retry = 0 2024-11-12T15:14:05,167 DEBUG [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/oldWALs 2024-11-12T15:14:05,167 INFO [RS:0;2b6d221c5cde:34071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2b6d221c5cde%2C34071%2C1731424341329.2b6d221c5cde%2C34071%2C1731424341329.regiongroup-1:(num 1731424432269) 2024-11-12T15:14:05,167 DEBUG [RS:0;2b6d221c5cde:34071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:05,167 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T15:14:05,167 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T15:14:05,168 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.ChoreService(370): Chore service for: regionserver/2b6d221c5cde:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T15:14:05,168 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T15:14:05,168 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T15:14:05,169 INFO [RS:0;2b6d221c5cde:34071 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34071 2024-11-12T15:14:05,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-12T15:14:05,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/2b6d221c5cde,34071,1731424341329 2024-11-12T15:14:05,236 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T15:14:05,236 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2b6d221c5cde,34071,1731424341329] 2024-11-12T15:14:05,246 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/2b6d221c5cde,34071,1731424341329 already deleted, retry=false 2024-11-12T15:14:05,246 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2b6d221c5cde,34071,1731424341329 expired; onlineServers=0 2024-11-12T15:14:05,246 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2b6d221c5cde,43085,1731424339322' ***** 2024-11-12T15:14:05,246 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T15:14:05,247 INFO [M:0;2b6d221c5cde:43085 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-12T15:14:05,253 INFO [M:0;2b6d221c5cde:43085 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2aafda76{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-12T15:14:05,254 INFO [M:0;2b6d221c5cde:43085 {}] server.AbstractConnector(383): Stopped ServerConnector@20f7d02c{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-12T15:14:05,254 INFO [M:0;2b6d221c5cde:43085 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:05,254 INFO [M:0;2b6d221c5cde:43085 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@e95be18{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-12T15:14:05,254 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-12T15:14:05,254 INFO [M:0;2b6d221c5cde:43085 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4cdc734c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:05,255 INFO [M:0;2b6d221c5cde:43085 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T15:14:05,255 INFO [M:0;2b6d221c5cde:43085 {}] master.HMaster(1770): Stopping master jetty server 2024-11-12T15:14:05,255 INFO [M:0;2b6d221c5cde:43085 {}] server.AbstractConnector(383): Stopped ServerConnector@629f2090{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-12T15:14:05,256 INFO [M:0;2b6d221c5cde:43085 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T15:14:05,256 DEBUG [M:0;2b6d221c5cde:43085 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T15:14:05,256 INFO [M:0;2b6d221c5cde:43085 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-12T15:14:05,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-12T15:14:05,256 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T15:14:05,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-12T15:14:05,256 INFO [M:0;2b6d221c5cde:43085 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;2b6d221c5cde:43085. 2024-11-12T15:14:05,256 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.large.0-1731424345038 {}] cleaner.HFileCleaner(306): Exit Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.large.0-1731424345038,5,FailOnTimeoutGroup] 2024-11-12T15:14:05,257 DEBUG [M:0;2b6d221c5cde:43085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T15:14:05,257 DEBUG [M:0;2b6d221c5cde:43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:05,257 DEBUG [M:0;2b6d221c5cde:43085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T15:14:05,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T15:14:05,257 DEBUG [M:0;2b6d221c5cde:43085 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T15:14:05,257 INFO [M:0;2b6d221c5cde:43085 {}] hbase.ChoreService(370): Chore service for: master/2b6d221c5cde:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T15:14:05,257 INFO [M:0;2b6d221c5cde:43085 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T15:14:05,258 DEBUG [M:0;2b6d221c5cde:43085 {}] master.HMaster(1795): Stopping service threads 2024-11-12T15:14:05,258 INFO [M:0;2b6d221c5cde:43085 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T15:14:05,258 INFO [M:0;2b6d221c5cde:43085 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T15:14:05,258 INFO [M:0;2b6d221c5cde:43085 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T15:14:05,259 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T15:14:05,260 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.small.0-1731424345040 {}] cleaner.HFileCleaner(306): Exit Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.small.0-1731424345040,5,FailOnTimeoutGroup] 2024-11-12T15:14:05,261 DEBUG [M:0;2b6d221c5cde:43085 {}] zookeeper.ZKUtil(347): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-12T15:14:05,261 WARN [M:0;2b6d221c5cde:43085 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T15:14:05,262 INFO [M:0;2b6d221c5cde:43085 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/.lastflushedseqids 2024-11-12T15:14:05,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741981_1157 (size=495) 2024-11-12T15:14:05,347 INFO [RS:0;2b6d221c5cde:34071 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T15:14:05,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:14:05,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34071-0x1012f9abd630001, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:14:05,347 INFO [RS:0;2b6d221c5cde:34071 {}] regionserver.HRegionServer(1031): Exiting; stopping=2b6d221c5cde,34071,1731424341329; zookeeper connection closed. 2024-11-12T15:14:05,348 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48372e29 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48372e29 2024-11-12T15:14:05,348 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T15:14:05,679 INFO [M:0;2b6d221c5cde:43085 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T15:14:05,679 INFO [M:0;2b6d221c5cde:43085 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T15:14:05,679 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T15:14:05,693 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:14:05,693 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:14:05,693 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T15:14:05,693 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:14:05,693 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=319.19 KB heapSize=375.20 KB 2024-11-12T15:14:05,712 DEBUG [M:0;2b6d221c5cde:43085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb4bdab8a5cf42c18138d277419173cc is 82, key is hbase:meta,,1/info:regioninfo/1731424345864/Put/seqid=0 2024-11-12T15:14:05,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741982_1158 (size=5672) 2024-11-12T15:14:05,718 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb4bdab8a5cf42c18138d277419173cc 2024-11-12T15:14:05,744 DEBUG [M:0;2b6d221c5cde:43085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21f8d5cb1dcb4aae8321107c1a082f41 is 2183, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1731424395861/Put/seqid=0 2024-11-12T15:14:05,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741983_1159 (size=26514) 2024-11-12T15:14:06,149 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.63 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21f8d5cb1dcb4aae8321107c1a082f41 2024-11-12T15:14:06,159 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21f8d5cb1dcb4aae8321107c1a082f41 2024-11-12T15:14:06,178 DEBUG [M:0;2b6d221c5cde:43085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8929f7fde96a471f915be8162919eff8 is 69, key is 2b6d221c5cde,34071,1731424341329/rs:state/1731424345074/Put/seqid=0 2024-11-12T15:14:06,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741984_1160 (size=5156) 2024-11-12T15:14:06,584 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8929f7fde96a471f915be8162919eff8 2024-11-12T15:14:06,597 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eb4bdab8a5cf42c18138d277419173cc as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb4bdab8a5cf42c18138d277419173cc 2024-11-12T15:14:06,604 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eb4bdab8a5cf42c18138d277419173cc, entries=8, sequenceid=667, filesize=5.5 K 2024-11-12T15:14:06,605 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21f8d5cb1dcb4aae8321107c1a082f41 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21f8d5cb1dcb4aae8321107c1a082f41 2024-11-12T15:14:06,610 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21f8d5cb1dcb4aae8321107c1a082f41 2024-11-12T15:14:06,610 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21f8d5cb1dcb4aae8321107c1a082f41, entries=69, sequenceid=667, filesize=25.9 K 2024-11-12T15:14:06,611 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8929f7fde96a471f915be8162919eff8 as hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8929f7fde96a471f915be8162919eff8 2024-11-12T15:14:06,615 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33269/user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8929f7fde96a471f915be8162919eff8, entries=1, sequenceid=667, filesize=5.0 K 2024-11-12T15:14:06,616 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(3140): Finished flush of dataSize ~319.19 KB/326851, heapSize ~374.90 KB/383896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 923ms, sequenceid=667, compaction requested=false 2024-11-12T15:14:06,617 INFO [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T15:14:06,618 DEBUG [M:0;2b6d221c5cde:43085 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731424445679Disabling compacts and flushes for region at 1731424445679Disabling writes for close at 1731424445693 (+14 ms)Obtaining lock to block concurrent updates at 1731424445693Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731424445693Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326851, getHeapSize=384136, getOffHeapSize=0, getCellsCount=746 at 1731424445693Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731424445694 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731424445694Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731424445712 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731424445712Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731424445723 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731424445744 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731424445744Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731424446159 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731424446178 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731424446178Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b3abb97: reopening flushed file at 1731424446595 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c14d19b: reopening flushed file at 1731424446604 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65468dd5: reopening flushed file at 1731424446610 (+6 ms)Finished flush of dataSize ~319.19 KB/326851, heapSize ~374.90 KB/383896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 923ms, sequenceid=667, compaction requested=false at 1731424446616 (+6 ms)Writing region close event to WAL at 1731424446617 (+1 ms)Closed at 1731424446617 2024-11-12T15:14:06,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741830_1006 (size=370941) 2024-11-12T15:14:06,620 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d8b46fc5-22fb-4923-ae65-215415c4c265/MasterData/WALs/2b6d221c5cde,43085,1731424339322/2b6d221c5cde%2C43085%2C1731424339322.2b6d221c5cde%2C43085%2C1731424339322.regiongroup-0.1731424343695 not finished, retry = 0 2024-11-12T15:14:06,723 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T15:14:06,723 INFO [M:0;2b6d221c5cde:43085 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T15:14:06,724 INFO [M:0;2b6d221c5cde:43085 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-12T15:14:06,724 INFO [M:0;2b6d221c5cde:43085 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43085 2024-11-12T15:14:06,724 INFO [M:0;2b6d221c5cde:43085 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T15:14:06,894 INFO [M:0;2b6d221c5cde:43085 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T15:14:06,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:14:06,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43085-0x1012f9abd630000, quorum=127.0.0.1:51996, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T15:14:06,909 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@cb44ad4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T15:14:06,911 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T15:14:06,911 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:06,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T15:14:06,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:06,913 WARN [BP-1953189042-172.17.0.3-1731424333465 heartbeating to localhost/127.0.0.1:33269 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T15:14:06,913 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T15:14:06,913 WARN [BP-1953189042-172.17.0.3-1731424333465 heartbeating to localhost/127.0.0.1:33269 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1953189042-172.17.0.3-1731424333465 (Datanode Uuid 6317909c-bf35-4176-afcd-944ba4e67199) service to localhost/127.0.0.1:33269 2024-11-12T15:14:06,913 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T15:14:06,915 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/data/data1/current/BP-1953189042-172.17.0.3-1731424333465 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T15:14:06,915 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/cluster_4a955086-6ea0-4762-0b1a-adc9d2084b01/data/data2/current/BP-1953189042-172.17.0.3-1731424333465 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T15:14:06,916 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T15:14:06,922 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c784fd0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T15:14:06,923 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2fdb04ed{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T15:14:06,923 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:06,924 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@282f8860{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T15:14:06,924 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3cb0c733{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:06,933 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T15:14:06,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T15:14:06,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-12T15:14:06,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@373f13ac{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T15:14:06,991 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5310ce69{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T15:14:06,991 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:06,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4db04e37{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T15:14:06,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2788ebc5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:06,999 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6449e4bf{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-12T15:14:07,000 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b8b49f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T15:14:07,000 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:07,000 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fac336c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T15:14:07,000 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61534992{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:07,005 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-12T15:14:07,005 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a9ddefe{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-12T15:14:07,005 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e5e639{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T15:14:07,005 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:07,006 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c57c6a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T15:14:07,006 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11a358cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:07,008 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-12T15:14:07,010 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-12T15:14:07,010 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-12T15:14:07,012 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-12T15:14:07,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6fc8b3aa{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-12T15:14:07,015 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37e8ed89{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T15:14:07,015 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T15:14:07,015 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a908e18{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-12T15:14:07,015 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26c56adc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/888e0bfc-e06d-b6ae-0e23-bf038f92e3b6/hadoop.log.dir/,STOPPED} 2024-11-12T15:14:07,016 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-12T15:14:07,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped